From 4f2553c5f95ac2948a0233e60db4514b917fb14b Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:21:43 -0700 Subject: [PATCH 1/9] =?UTF-8?q?refactor(agent):=20compact=20iron=5Fproxy?= =?UTF-8?q?=20=E2=80=94=20inline=20single-use=20download/token=20helpers,?= =?UTF-8?q?=20unify=20listen=20readers,=20contextlib.suppress?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/proxy_sources/iron_proxy.py | 342 ++++++++---------------------- 1 file changed, 94 insertions(+), 248 deletions(-) diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index 74320f89a9..5a055c2e82 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -24,6 +24,7 @@ import threading import time import urllib.error import urllib.request +from contextlib import suppress from dataclasses import dataclass, field, replace from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -33,7 +34,6 @@ logger = logging.getLogger(__name__) # Pinned upstream version. Never auto-resolve "latest": the YAML schema may change between # releases and updates must be deliberate. _IRON_PROXY_VERSION = "0.39.0" - _IRON_PROXY_RELEASE_BASE = f"https://github.com/ironsh/iron-proxy/releases/download/v{_IRON_PROXY_VERSION}" _IRON_PROXY_CHECKSUM_NAME = "checksums.txt" # Detached signature + signing key for optional GPG verification of checksums.txt (SHA-256 @@ -49,8 +49,7 @@ _STARTUP_GRACE_SECONDS = 5 # ruleset. Bearer key minted at setup, stored 0600 at /management.token, injected into # the daemon env under this name; v0.39 refuses to start if the named var is empty. _MGMT_API_KEY_ENV = "HERMES_IRON_PROXY_MGMT_KEY" -# tunnel_port is CONNECT/MITM, +1 is plain-HTTP forward, +2 is management. -_MGMT_PORT_OFFSET = 2 +_MGMT_PORT_OFFSET = 2 # tunnel_port is CONNECT/MITM, +1 is plain-HTTP forward, +2 is management _MGMT_RELOAD_TIMEOUT = 15 # HTTPS_PROXY semantics use a single CONNECT tunnel, so only the tunnel listener is exposed. @@ -79,19 +78,15 @@ _BEARER_PROVIDERS: Dict[str, Tuple[str, ...]] = { # case-insensitive). ``aliases`` are interchangeable env names for the SAME credential and # MUST collapse into one mapping: every rule is ``require: true`` and two require-rules on one # host would reject each other's requests. The sandbox gets the token under every name. +# Authorization is also matched for Anthropic/Azure so an SDK sending the token as Bearer still +# swaps; Gemini's ``?key=`` SDK style is covered by match_query. _HEADER_AUTH_PROVIDERS: Dict[str, Dict[str, Tuple[str, ...]]] = { - # Authorization also matched so an SDK sending the token as Bearer still swaps. - "ANTHROPIC_API_KEY": { - "hosts": ("api.anthropic.com",), - "match_headers": ("x-api-key", "Authorization"), - "aliases": (), - }, + "ANTHROPIC_API_KEY": {"hosts": ("api.anthropic.com",), "match_headers": ("x-api-key", "Authorization"), "aliases": ()}, "AZURE_OPENAI_API_KEY": { "hosts": ("*.openai.azure.com", "*.cognitiveservices.azure.com", "*.services.ai.azure.com"), "match_headers": ("api-key", "Authorization"), "aliases": (), }, - # ``?key=`` SDK style is covered by match_query. "GEMINI_API_KEY": { "hosts": ("generativelanguage.googleapis.com",), "match_headers": ("x-goog-api-key",), @@ -108,14 +103,9 @@ _NON_BEARER_PROVIDERS: Tuple[str, ...] = ( # Default SSRF deny list for outbound traffic (docs promise: cloud metadata IPs refused # regardless of allowlist). Callers may pass [] to disable (hermetic tests only). _DEFAULT_UPSTREAM_DENY_CIDRS: Tuple[str, ...] = ( - "127.0.0.0/8", # IPv4 loopback - "::1/128", # IPv6 loopback - "169.254.0.0/16", # IPv4 link-local incl. AWS/GCP/Azure IMDS - "fe80::/10", # IPv6 link-local - "10.0.0.0/8", # RFC1918 - "172.16.0.0/12", # RFC1918 - "192.168.0.0/16", # RFC1918 - "fc00::/7", # IPv6 ULA + "127.0.0.0/8", "::1/128", # loopback v4 / v6 + "169.254.0.0/16", "fe80::/10", # link-local incl. AWS/GCP/Azure IMDS + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", # RFC1918 + IPv6 ULA "::ffff:0:0/96", # IPv4-mapped IPv6 — else ::ffff:169.254.169.254 bypasses IMDS deny "100.64.0.0/10", # RFC6598 CGNAT (AWS VPC shared services, k8s pod nets) "198.18.0.0/15", # RFC2544 benchmark range @@ -184,14 +174,12 @@ class TokenMapping: def _hermes_bin_dir() -> Path: from hermes_constants import get_hermes_home - return get_hermes_home() / "bin" def _proxy_state_dir_ro() -> Path: """Proxy state dir without creating it (status probes, pidfile reads).""" from hermes_constants import get_hermes_home - return get_hermes_home() / "proxy" @@ -200,10 +188,8 @@ def _proxy_state_dir() -> Path: so a pre-existing dir with a slack umask gets tightened.""" d = _proxy_state_dir_ro() d.mkdir(parents=True, exist_ok=True) - try: + with suppress(OSError): # Windows no-op / shared fs we don't own; files still get explicit perms d.chmod(0o700) - except OSError: - pass # Windows no-op / shared fs we don't own; files still get explicit perms return d @@ -286,25 +272,21 @@ def install_iron_proxy(*, force: bool = False) -> Path: # A freshly-installed binary must re-probe --version on the next get_status(). _VERSION_CACHE.pop(str(target), None) - logger.info("Installed iron-proxy %s at %s", _IRON_PROXY_VERSION, target) return target -def _http_download(url: str, dest: Path) -> None: +def _release_asset(name: str, dest: Path) -> None: + """Download one pinned-release asset to ``dest``; RuntimeError on any URL error.""" + url = f"{_IRON_PROXY_RELEASE_BASE}/{name}" req = urllib.request.Request(url, headers={"User-Agent": "hermes-agent"}) try: - with urllib.request.urlopen(req, timeout=_DOWNLOAD_TIMEOUT) as resp: # noqa: S310 - with open(dest, "wb") as f: - shutil.copyfileobj(resp, f) + with urllib.request.urlopen(req, timeout=_DOWNLOAD_TIMEOUT) as resp, open(dest, "wb") as f: # noqa: S310 + shutil.copyfileobj(resp, f) except urllib.error.URLError as exc: raise RuntimeError(f"Failed to download {url}: {exc}") from exc -def _release_asset(name: str, dest: Path) -> None: - _http_download(f"{_IRON_PROXY_RELEASE_BASE}/{name}", dest) - - def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: """Best-effort GPG verification of checksums.txt in an ephemeral keyring. Returns False (with a warning) when gpg or the signature assets are unavailable — SHA-256 stays enforced and gpg @@ -351,8 +333,7 @@ def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: def _expected_sha256(checksum_file: Path, asset_name: str) -> str: """Parse ``sha256sum`` output (`` ``).""" - text = checksum_file.read_text(encoding="utf-8", errors="replace") - for line in text.splitlines(): + for line in checksum_file.read_text(encoding="utf-8", errors="replace").splitlines(): parts = line.strip().split() if len(parts) >= 2 and parts[-1] == asset_name: return parts[0] @@ -415,10 +396,8 @@ def _write_private_file(path: Path, data: bytes) -> None: defends against a planted symlink; fchmod tightens a pre-existing file.""" fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC | _O_NOFOLLOW, 0o600) try: - try: + with suppress(OSError, AttributeError): os.fchmod(fd, 0o600) - except (OSError, AttributeError): - pass os.write(fd, data) finally: os.close(fd) @@ -441,20 +420,16 @@ def ensure_ca_cert(*, force: bool = False) -> Tuple[Path, Path]: return ca_crt, ca_key if shutil.which("openssl") is None: raise RuntimeError( - "openssl not found on PATH. Install OpenSSL (apt: `openssl`, " - "brew: `openssl`) to generate the iron-proxy CA cert." + "openssl not found on PATH. Install OpenSSL (apt: `openssl`, brew: `openssl`) to generate the iron-proxy CA cert." ) with tempfile.TemporaryDirectory(prefix="hermes-proxy-ca-") as tmpdir: tmp_key, tmp_crt = Path(tmpdir) / "ca.key", Path(tmpdir) / "ca.crt" _run(["openssl", "genrsa", "-out", str(tmp_key), "4096"], timeout=60, check=True) _run([ - "openssl", "req", "-x509", "-new", "-nodes", - "-key", str(tmp_key), - "-sha256", "-days", "3650", - "-subj", "/CN=hermes iron-proxy CA", - "-addext", "basicConstraints=critical,CA:TRUE", - "-addext", "keyUsage=critical,keyCertSign", + "openssl", "req", "-x509", "-new", "-nodes", "-key", str(tmp_key), + "-sha256", "-days", "3650", "-subj", "/CN=hermes iron-proxy CA", + "-addext", "basicConstraints=critical,CA:TRUE", "-addext", "keyUsage=critical,keyCertSign", "-out", str(tmp_crt), ], timeout=60, check=True) @@ -495,10 +470,6 @@ def ensure_management_token(*, force: bool = False) -> str: return token -def _read_management_token() -> Optional[str]: - return _read_text_or_none(_proxy_state_dir_ro() / "management.token") - - def _load_proxy_yaml(cfg: Path): """Parsed proxy.yaml, or ``{}`` when missing/unreadable/PyYAML absent.""" try: @@ -523,34 +494,29 @@ def _parse_listen(listen) -> Optional[Tuple[str, int]]: return (host or "127.0.0.1", port) +def _config_listen(section: str, *keys: str, config_path: Optional[Path] = None) -> Optional[Tuple[str, int]]: + """``(host, port)`` from the first truthy ``proxy.yaml[section][key]``, or None.""" + block = _load_proxy_yaml(config_path or (_proxy_state_dir_ro() / "proxy.yaml")).get(section) or {} + return _parse_listen(next((block[k] for k in keys if block.get(k)), "")) + + def _read_management_listen_from_config(config_path: Optional[Path] = None) -> Optional[Tuple[str, int]]: - """``(host, port)`` of the management listener, if configured.""" - data = _load_proxy_yaml(config_path or (_proxy_state_dir_ro() / "proxy.yaml")) - return _parse_listen((data.get("management") or {}).get("listen") or "") - - -def _read_http_listen_from_config() -> Optional[Tuple[str, int]]: - """``(host, port)`` of the sandbox-facing listener: ``tunnel_listen`` (CONNECT/MITM), falling - back to ``http_listen`` for configs written before the listener-role split.""" - proxy_block = _load_proxy_yaml(_proxy_state_dir_ro() / "proxy.yaml").get("proxy") or {} - return _parse_listen(proxy_block.get("tunnel_listen") or proxy_block.get("http_listen") or "") + return _config_listen("management", "listen", config_path=config_path) def _probe_target() -> Tuple[str, int]: """Configured bind host/port to probe — on Linux that's the docker bridge, where a loopback - connect would report a healthy daemon as down.""" - return _read_http_listen_from_config() or ("127.0.0.1", _DEFAULT_TUNNEL_PORT) + connect would report a healthy daemon as down. ``tunnel_listen`` (CONNECT/MITM) wins, falling + back to ``http_listen`` for configs written before the listener-role split.""" + return _config_listen("proxy", "tunnel_listen", "http_listen") or ("127.0.0.1", _DEFAULT_TUNNEL_PORT) # Management-API error status -> operator message (422 = validation rejected, running ruleset # unchanged; 401 = daemon started with a different management.token). _RELOAD_HTTP_ERRORS = { 422: "iron-proxy rejected the new config (validation failed; the running ruleset is unchanged): {body}", - 401: ( - "management API rejected our key (401). The running " - "daemon was started with a different management.token — " - "run `hermes egress restart`." - ), + 401: "management API rejected our key (401). The running daemon was started with a different management.token — " + "run `hermes egress restart`.", } @@ -563,23 +529,16 @@ def reload_proxy() -> bool: mgmt = _read_management_listen_from_config() if mgmt is None: raise RuntimeError( - "The generated proxy.yaml has no management listener (written " - "before reload support). Re-run `hermes egress setup` and use " - "`hermes egress restart` this one time." + "The generated proxy.yaml has no management listener (written before reload support). " + "Re-run `hermes egress setup` and use `hermes egress restart` this one time." ) - token = _read_management_token() + token = _read_text_or_none(_proxy_state_dir_ro() / "management.token") if not token: - raise RuntimeError( - "management.token is missing — re-run `hermes egress setup`, " - "then `hermes egress restart`." - ) + raise RuntimeError("management.token is missing — re-run `hermes egress setup`, then `hermes egress restart`.") host, port = mgmt req = urllib.request.Request( - f"http://{host}:{port}/v1/reload", - method="POST", - headers={"Authorization": f"Bearer {token}"}, - data=b"", + f"http://{host}:{port}/v1/reload", method="POST", headers={"Authorization": f"Bearer {token}"}, data=b"", ) try: with urllib.request.urlopen(req, timeout=_MGMT_RELOAD_TIMEOUT) as resp: @@ -597,8 +556,7 @@ def reload_proxy() -> bool: # A daemon started from a pre-management config is alive but has no listener. raise RuntimeError( f"could not reach the management API at {host}:{port} ({exc}). " - "If the daemon was started before reload support, run " - "`hermes egress restart` once." + "If the daemon was started before reload support, run `hermes egress restart` once." ) from exc @@ -612,9 +570,8 @@ def _default_http_listen(tunnel_port: int) -> List[str]: if bridge_ip and bridge_ip != "127.0.0.1": return [f"{bridge_ip}:{tunnel_port}"] logger.warning( - "No docker bridge (docker0) detected — binding iron-proxy to " - "loopback only. Docker sandboxes will NOT be able to reach " - "the proxy until it is restarted with docker running." + "No docker bridge (docker0) detected — binding iron-proxy to loopback only. " + "Docker sandboxes will NOT be able to reach the proxy until it is restarted with docker running." ) return [f"127.0.0.1:{tunnel_port}"] @@ -629,43 +586,28 @@ def _detect_docker_bridge_ip() -> Optional[str]: return None if res.returncode != 0: return None - # Expected: ": docker0 inet 172.17.0.1/16 ..." — first inet token wins. + # Expected: ": docker0 inet 172.17.0.1/16 ..." — first inet token (per line) wins. candidate = None - for line in res.stdout.splitlines(): - parts = line.split() - hits = [parts[i + 1] for i, tok in enumerate(parts[:-1]) if tok == "inet"] - if hits: - candidate = hits[0].split("/")[0] + for parts in (line.split() for line in res.stdout.splitlines()): + if "inet" in parts[:-1]: + candidate = parts[parts.index("inet") + 1].split("/")[0] break if not candidate: return None - try: addr = ipaddress.IPv4Address(candidate) except (ipaddress.AddressValueError, ValueError): return None - if ( - addr.is_unspecified or addr.is_loopback or addr.is_multicast - or addr.is_reserved or addr.is_link_local or addr.is_global - ): - logger.warning( - "Refusing suspicious docker bridge IP %s reported by `ip`; " - "skipping bridge bind.", candidate, - ) + if addr.is_unspecified or addr.is_loopback or addr.is_multicast or addr.is_reserved or addr.is_link_local or addr.is_global: + logger.warning("Refusing suspicious docker bridge IP %s reported by `ip`; skipping bridge bind.", candidate) return None return str(addr) def build_proxy_config( - *, - mappings: List[TokenMapping], - ca_cert: Path, - ca_key: Path, - tunnel_port: int = _DEFAULT_TUNNEL_PORT, - audit_log: Optional[Path] = None, - allowed_hosts: Optional[List[str]] = None, - upstream_deny_cidrs: Optional[List[str]] = None, - http_listen: Optional[List[str]] = None, + *, mappings: List[TokenMapping], ca_cert: Path, ca_key: Path, tunnel_port: int = _DEFAULT_TUNNEL_PORT, + audit_log: Optional[Path] = None, allowed_hosts: Optional[List[str]] = None, + upstream_deny_cidrs: Optional[List[str]] = None, http_listen: Optional[List[str]] = None, ) -> Dict: """Build the iron-proxy YAML config dict (schema as of v0.39.0). Real secrets are read from iron-proxy's OWN env (``source: {type: env}``); the sandbox never sees them. @@ -685,11 +627,8 @@ def build_proxy_config( { "source": {"type": "env", "var": m.real_env_name}, "replace": { - "proxy_value": m.proxy_token, - "match_headers": list(m.match_headers or ("Authorization",)), - "match_query": True, - "match_body": False, - "require": True, + "proxy_value": m.proxy_token, "match_headers": list(m.match_headers or ("Authorization",)), + "match_query": True, "match_body": False, "require": True, }, "rules": [{"host": h} for h in m.upstream_hosts], } @@ -710,8 +649,7 @@ def build_proxy_config( # Both bind the docker bridge on Linux / loopback on Docker Desktop — NEVER 0.0.0.0. "tunnel_listen": primary_listen, "http_listen": f"{bind_host}:{tunnel_port + 1}", - # Direct-TLS listener is not exposed. - "https_listen": "127.0.0.1:0", + "https_listen": "127.0.0.1:0", # direct-TLS listener is not exposed "max_request_body_bytes": 16 * 1024 * 1024, "max_response_body_bytes": 0, "upstream_response_header_timeout": "120s", @@ -787,11 +725,8 @@ def write_mappings(mappings: List[TokenMapping]) -> Path: "version": 1, "tokens": [ { - "proxy_token": m.proxy_token, - "env_name": m.real_env_name, - "upstream_hosts": list(m.upstream_hosts), - "match_headers": list(m.match_headers), - "alias_env_names": list(m.alias_env_names), + "proxy_token": m.proxy_token, "env_name": m.real_env_name, "upstream_hosts": list(m.upstream_hosts), + "match_headers": list(m.match_headers), "alias_env_names": list(m.alias_env_names), } for m in mappings ], @@ -842,13 +777,7 @@ def discover_provider_mappings(*, available_env_names: Optional[List[str]] = Non for n, s in _HEADER_AUTH_PROVIDERS.items() ] return [ - TokenMapping( - proxy_token=mint_proxy_token(prefix=env_name.lower().replace("_api_key", "")), - real_env_name=env_name, - upstream_hosts=hosts, - match_headers=headers, - alias_env_names=aliases, - ) + TokenMapping(mint_proxy_token(prefix=env_name.lower().replace("_api_key", "")), env_name, hosts, headers, aliases) for env_name, hosts, headers, aliases in specs if env_name in names or any(a in names for a in aliases) ] @@ -860,9 +789,7 @@ def discover_uncovered_providers(*, available_env_names: Optional[List[str]] = N return [n for n in _NON_BEARER_PROVIDERS if n in names] -def merge_mappings( - *, existing: List[TokenMapping], discovered: List[TokenMapping], rotate: bool = False, -) -> List[TokenMapping]: +def merge_mappings(*, existing: List[TokenMapping], discovered: List[TokenMapping], rotate: bool = False) -> List[TokenMapping]: """Combine existing and freshly discovered mappings. Tokens already in ``existing`` are preserved (containers baked with them keep working) while hosts/headers/aliases are refreshed from ``discovered``; ``rotate=True`` re-mints everything; undiscovered providers are dropped.""" @@ -939,41 +866,26 @@ def _pid_alive(pid: int) -> bool: # Strong proof: nonce from this process's start_proxy and/or the on-disk sibling file. nonce_candidates = [n for n in dict.fromkeys((_proxy_nonce, _read_persisted_nonce())) if n] if nonce_candidates: - try: + with suppress(OSError): env_bytes = Path(f"/proc/{pid}/environ").read_bytes() if any(f"{_HERMES_IRON_PROXY_NONCE_ENV}={n}".encode() in env_bytes for n in nonce_candidates): return True - except OSError: - pass - - try: + with suppress(OSError): cmdline_path = Path(f"/proc/{pid}/cmdline") if cmdline_path.exists(): argv0 = cmdline_path.read_bytes().split(b"\x00")[0].decode("utf-8", errors="ignore") return os.path.basename(argv0).startswith("iron-proxy") - except OSError: - pass - - # macOS / non-Linux fallback. - try: + with suppress(OSError, subprocess.TimeoutExpired): # macOS / non-Linux fallback res = _run(["ps", "-p", str(pid), "-o", "comm="], timeout=2, text=True) if res.returncode == 0: return os.path.basename((res.stdout or "").strip()).startswith("iron-proxy") - except (OSError, subprocess.TimeoutExpired): - pass - # Exotic platforms: if the OS says alive, believe it. return True def start_proxy( - *, - binary: Optional[Path] = None, - config_path: Optional[Path] = None, - extra_env: Optional[Dict[str, str]] = None, - install_if_missing: bool = True, - refresh_secrets_from_bitwarden: bool = False, - bitwarden_config: Optional[Dict] = None, + *, binary: Optional[Path] = None, config_path: Optional[Path] = None, extra_env: Optional[Dict[str, str]] = None, + install_if_missing: bool = True, refresh_secrets_from_bitwarden: bool = False, bitwarden_config: Optional[Dict] = None, ) -> ProxyStatus: """Spawn iron-proxy as a managed background subprocess (idempotent if already running). ``refresh_secrets_from_bitwarden=True`` re-fetches upstream secrets from BWS at startup — @@ -992,9 +904,7 @@ def start_proxy( # Minimal env: os.environ.copy() would expose every operator secret via /proc//environ. env = _build_proxy_subprocess_env( - extra_env=extra_env, - refresh_from_bitwarden=refresh_secrets_from_bitwarden, - bitwarden_config=bitwarden_config, + extra_env=extra_env, refresh_from_bitwarden=refresh_secrets_from_bitwarden, bitwarden_config=bitwarden_config, ) # v0.39 validates api_key_env is non-empty when management.listen is set. if _read_management_listen_from_config(cfg) is not None: @@ -1049,12 +959,10 @@ def start_proxy( # Process may have died right at deadline. if proc.poll() is not None: raise _exited_error() - # Alive-but-not-listening is a failure: an orphan holding the port breaks every restart. if not listening: raise _abort( - f"iron-proxy did not bind {probe_host}:{tunnel_port} within " - f"{_STARTUP_GRACE_SECONDS}s. Process was killed. ", + f"iron-proxy did not bind {probe_host}:{tunnel_port} within {_STARTUP_GRACE_SECONDS}s. Process was killed. ", kill=True, ) @@ -1069,9 +977,7 @@ def _spawn_daemon(bin_path: Path, cfg: Path, env: Dict[str, str], log_path: Path try: log_fd = _open_private_append(log_path, strict_chmod=False) except OSError as exc: - raise RuntimeError( - f"Refusing to write iron-proxy log {log_path}: {exc}. Remove that path manually and retry." - ) from exc + raise RuntimeError(f"Refusing to write iron-proxy log {log_path}: {exc}. Remove that path manually and retry.") from exc if not _fd_owned_by_us(log_fd): st = os.fstat(log_fd) os.close(log_fd) @@ -1086,10 +992,8 @@ def _spawn_daemon(bin_path: Path, cfg: Path, env: Dict[str, str], log_path: Path except OSError as exc: raise RuntimeError(f"failed to spawn iron-proxy: {exc}") from exc finally: - try: + with suppress(OSError): os.close(log_fd) - except OSError: - pass def _await_listening(proc: "subprocess.Popen", host: str, port: int, *, on_exit) -> bool: @@ -1117,17 +1021,14 @@ def _write_pidfile_safely(pidfile: Path, pid: int) -> None: existing_pid = _read_pid() if existing_pid and _pid_alive(existing_pid): raise RuntimeError( - f"Another iron-proxy start appears to be in progress " - f"(pidfile {pidfile} -> pid {existing_pid}). " + f"Another iron-proxy start appears to be in progress (pidfile {pidfile} -> pid {existing_pid}). " f"Run `hermes egress stop` if that proxy is stuck." ) pidfile.unlink(missing_ok=True) fd = os.open(str(pidfile), open_flags, 0o600) except OSError as exc: # ELOOP from a planted symlink at the pidfile path. - raise RuntimeError( - f"Refusing to write pidfile {pidfile}: {exc}. Remove that path manually and retry." - ) from exc + raise RuntimeError(f"Refusing to write pidfile {pidfile}: {exc}. Remove that path manually and retry.") from exc try: if not _fd_owned_by_us(fd): @@ -1138,10 +1039,8 @@ def _write_pidfile_safely(pidfile: Path, pid: int) -> None: # Best-effort nonce sibling (0o600); without it stop falls back to argv0 matching. if _proxy_nonce: - try: + with suppress(OSError): _write_private_file(pidfile.with_suffix(".nonce"), _proxy_nonce.encode("utf-8")) - except OSError: - pass def _kill_and_wait(proc: "subprocess.Popen", *, grace_seconds: int = 2) -> None: @@ -1153,21 +1052,14 @@ def _kill_and_wait(proc: "subprocess.Popen", *, grace_seconds: int = 2) -> None: try: proc.wait(timeout=grace_seconds) except subprocess.TimeoutExpired: - try: + with suppress(OSError): proc.kill() - except OSError: - pass - try: + with suppress(subprocess.TimeoutExpired): proc.wait(timeout=grace_seconds) - except subprocess.TimeoutExpired: - pass def _build_proxy_subprocess_env( - *, - extra_env: Optional[Dict[str, str]] = None, - refresh_from_bitwarden: bool = False, - bitwarden_config: Optional[Dict] = None, + *, extra_env: Optional[Dict[str, str]] = None, refresh_from_bitwarden: bool = False, bitwarden_config: Optional[Dict] = None, ) -> Dict[str, str]: """Minimal env for the daemon: allowlisted infra vars + the secrets named in mappings. With ``refresh_from_bitwarden`` and a populated ``bitwarden_config``, secrets are fetched from BWS @@ -1191,16 +1083,13 @@ def _build_proxy_subprocess_env( break if refresh_from_bitwarden and bitwarden_config: - _refresh_secrets_from_bitwarden( - env, needed, bitwarden_config, bool(bitwarden_config.get("allow_env_fallback")), - ) + _refresh_secrets_from_bitwarden(env, needed, bitwarden_config, bool(bitwarden_config.get("allow_env_fallback"))) # Caller overrides win (wizard test secrets), then strip proxy-recursion vars regardless. if extra_env: env.update(extra_env) for name in _PROXY_SUBPROCESS_ENV_STRIP: env.pop(name, None) - env.setdefault("NO_COLOR", "1") return env @@ -1213,9 +1102,7 @@ def _bitwarden_shortfall(allow_env_fallback: bool, error: str, warning: str, *ar logger.warning(warning, *args) -def _refresh_secrets_from_bitwarden( - env: Dict[str, str], needed: set, bitwarden_config: Dict, allow_env_fallback: bool, -) -> None: +def _refresh_secrets_from_bitwarden(env: Dict[str, str], needed: set, bitwarden_config: Dict, allow_env_fallback: bool) -> None: """Overwrite ``env[needed]`` with fresh BWS values (uncached fetch). Only mapped names are injected so unrelated BWS secrets never leak into the daemon env.""" try: @@ -1227,36 +1114,26 @@ def _refresh_secrets_from_bitwarden( # Don't interpolate access_token_name — CodeQL treats config values as tainted. _bitwarden_shortfall( allow_env_fallback, - "credential_source=bitwarden but the access-token " - "env or project_id is empty. Either set both, " - "switch to credential_source: env, or set " - "`proxy.allow_env_fallback: true` to opt into " + "credential_source=bitwarden but the access-token env or project_id is empty. Either set both, " + "switch to credential_source: env, or set `proxy.allow_env_fallback: true` to opt into " "the legacy fallback behaviour.", - "credential_source=bitwarden but access-token env or " - "project_id is empty — proxy will fall back to parent env " - "(allow_env_fallback=true).", + "credential_source=bitwarden but access-token env or project_id is empty — " + "proxy will fall back to parent env (allow_env_fallback=true).", ) return secrets, warnings = bw.fetch_bitwarden_secrets( - access_token=access_token, - project_id=project_id, - cache_ttl_seconds=0, - use_cache=False, + access_token=access_token, project_id=project_id, cache_ttl_seconds=0, use_cache=False, ) except ImportError as exc: # A dependency vanishing between setup and restart must not silently degrade. if not allow_env_fallback: raise RuntimeError( - "Bitwarden refresh module unavailable at proxy start " - "(credential_source=bitwarden with " - "proxy.allow_env_fallback: false). Either fix the " - "import, switch to credential_source: env, or set " - "`proxy.allow_env_fallback: true` to opt into the " - "legacy fallback behaviour." + "Bitwarden refresh module unavailable at proxy start (credential_source=bitwarden with " + "proxy.allow_env_fallback: false). Either fix the import, switch to credential_source: env, or set " + "`proxy.allow_env_fallback: true` to opt into the legacy fallback behaviour." ) from exc logger.warning( - "Bitwarden refresh module unavailable at proxy start, " - "falling back to parent env (allow_env_fallback=true): %s", + "Bitwarden refresh module unavailable at proxy start, falling back to parent env (allow_env_fallback=true): %s", exc, ) return @@ -1268,23 +1145,17 @@ def _refresh_secrets_from_bitwarden( if missing: _bitwarden_shortfall( allow_env_fallback, - f"Bitwarden refresh did not return secrets for " - f"{missing}. Either add the secrets to your BWS " - f"project, switch to credential_source: env via " - f"`hermes egress setup --no-bitwarden`, or set " - f"`proxy.allow_env_fallback: true` in config.yaml " - f"to opt into the legacy host-env fallback.", - "Bitwarden refresh did not return secrets for %s — " - "falling back to host env for those names " + f"Bitwarden refresh did not return secrets for {missing}. Either add the secrets to your BWS " + f"project, switch to credential_source: env via `hermes egress setup --no-bitwarden`, or set " + f"`proxy.allow_env_fallback: true` in config.yaml to opt into the legacy host-env fallback.", + "Bitwarden refresh did not return secrets for %s — falling back to host env for those names " "(allow_env_fallback=true).", missing, ) # Log only the count: the taint analyzer can't tell bws status text is non-secret. if warnings: logger.warning( - "Bitwarden refresh produced %d warning(s); " - "run `hermes secrets bitwarden status` for detail.", - len(warnings), + "Bitwarden refresh produced %d warning(s); run `hermes secrets bitwarden status` for detail.", len(warnings), ) @@ -1292,10 +1163,8 @@ def _forget_daemon() -> None: """Best-effort removal of pidfile + persisted nonce, and drop the in-process nonce.""" global _proxy_nonce _pidfile().unlink(missing_ok=True) - try: + with suppress(OSError): _persisted_nonce_path().unlink() - except OSError: - pass _proxy_nonce = None @@ -1308,7 +1177,6 @@ def stop_proxy() -> bool: # Capture starttime BEFORE signalling: if the pid is recycled mid-wait, abort the SIGKILL. starttime_before = _pid_proc_starttime(pid) - try: os.kill(pid, signal.SIGTERM) except ProcessLookupError: @@ -1329,10 +1197,8 @@ def stop_proxy() -> bool: if recycled: logger.warning("iron-proxy pid=%s appears recycled before SIGKILL; not killing.", pid) else: - try: + with suppress(ProcessLookupError): os.kill(pid, _KILL_SIGNAL) - except ProcessLookupError: - pass _forget_daemon() logger.info("Stopped iron-proxy pid=%s", pid) @@ -1343,29 +1209,24 @@ def get_status() -> ProxyStatus: """Snapshot the proxy state without side effects (called per Docker container create).""" status = ProxyStatus() probe_host, status.tunnel_port = _probe_target() - binary = find_iron_proxy(install_if_missing=False) if binary: status.binary_path = binary status.binary_version = iron_proxy_version(binary) - state = _proxy_state_dir_ro() cfg, ca = state / "proxy.yaml", state / "ca.crt" status.config_path = cfg if cfg.exists() else None status.ca_cert_path = ca if ca.exists() else None - pid = _read_pid() if pid and _pid_alive(pid): status.pid = pid status.listening = _port_listening(probe_host, status.tunnel_port) - return status def _port_listening(host: str, port: int) -> bool: """Cheap TCP connect probe — True iff something accepts on host:port.""" import socket - try: with socket.create_connection((host, port), timeout=0.5): return True @@ -1391,24 +1252,9 @@ def _reset_for_tests() -> None: __all__ = [ - "ProxyStatus", - "TokenMapping", - "build_proxy_config", - "discover_provider_mappings", - "discover_uncovered_providers", - "ensure_audit_log", - "ensure_ca_cert", - "ensure_management_token", - "find_iron_proxy", - "get_status", - "install_iron_proxy", - "iron_proxy_version", - "load_mappings", - "merge_mappings", - "mint_proxy_token", - "reload_proxy", - "start_proxy", - "stop_proxy", - "write_mappings", - "write_proxy_config", + "ProxyStatus", "TokenMapping", "build_proxy_config", "discover_provider_mappings", + "discover_uncovered_providers", "ensure_audit_log", "ensure_ca_cert", "ensure_management_token", + "find_iron_proxy", "get_status", "install_iron_proxy", "iron_proxy_version", "load_mappings", + "merge_mappings", "mint_proxy_token", "reload_proxy", "start_proxy", "stop_proxy", + "write_mappings", "write_proxy_config", ] From 1a1716ee6434e37a0f60ad2c18a27314e13f7376 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:24:14 -0700 Subject: [PATCH 2/9] =?UTF-8?q?refactor(agent):=20compact=20outbound=5Fweb?= =?UTF-8?q?hooks=20=E2=80=94=20fold=20list-check=20into=20iter=5Fconfigure?= =?UTF-8?q?d=5Ftargets,=20tighten=20delivery=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/outbound_webhooks.py | 167 +++++++++++-------------------------- 1 file changed, 49 insertions(+), 118 deletions(-) diff --git a/agent/outbound_webhooks.py b/agent/outbound_webhooks.py index 7fcef908f1..4604504147 100644 --- a/agent/outbound_webhooks.py +++ b/agent/outbound_webhooks.py @@ -1,31 +1,14 @@ -"""Outbound webhook notifications. +"""Outbound webhook notifications: ``hooks.outbound`` in config.yaml -> notify-only callbacks +on the plugin hook manager, so every ``invoke_hook()`` site can push lifecycle events to +external HTTP endpoints (outbound mirror of ``gateway/platforms/webhook.py``). -Reads ``hooks.outbound:`` from config.yaml and registers notify-only callbacks on -the plugin hook manager, so every ``invoke_hook()`` site can push lifecycle events -to external HTTP endpoints. Outbound mirror of ``gateway/platforms/webhook.py``. - -* Delivery is fire-and-forget through a bounded queue and one daemon worker - thread; callbacks serialize, enqueue, and return ``None`` immediately, so a - target can never block a tool call or influence agent flow. -* Payloads are HMAC-SHA256 signed (``X-Hermes-Signature-256: sha256=`` over - the raw body) when a secret is configured. -* No consent prompt (no code runs on this machine); ``HERMES_SAFE_MODE=1`` still - skips registration. Registration is idempotent. - -Config:: - - hooks: - outbound: - - url: https://ci.example.com/hermes-events - events: [on_session_end, subagent_stop] - secret_env: HERMES_OUTBOUND_WEBHOOK_SECRET # or inline ``secret`` - matcher: "terminal|delegate_task" # pre/post_tool_call only - timeout: 10 # seconds, clamped to [1, 60] - name: ci-notify - -POST body: ``{hook_event_name, profile, tool_name, tool_input, session_id, cwd, -extra, delivery_id, timestamp}``. Headers: ``Content-Type``, ``User-Agent``, -``X-Hermes-Event``, ``X-Hermes-Delivery``, ``X-Hermes-Signature-256`` (if secret). +* Fire-and-forget: callbacks serialize, enqueue on a bounded queue and return ``None``; one + daemon worker POSTs, so a target can never block a tool call or influence agent flow. +* HMAC-SHA256 signed (``X-Hermes-Signature-256: sha256=`` over the raw body) when a + secret is configured. ``HERMES_SAFE_MODE=1`` skips registration; registration is idempotent. +* Entry keys: url, events, secret_env|secret, matcher (pre/post_tool_call only), timeout + (clamped to [1, 60]), name. Body: ``{hook_event_name, profile, tool_name, tool_input, + session_id, cwd, extra, delivery_id, timestamp}``. """ from __future__ import annotations @@ -68,9 +51,7 @@ QUEUE_MAX_SIZE = 256 _registered: Set[Tuple[str, str, str]] = set() _registered_lock = threading.Lock() -_delivery_queue: "queue.Queue[Optional[Dict[str, Any]]]" = queue.Queue( - maxsize=QUEUE_MAX_SIZE -) +_delivery_queue: "queue.Queue[Optional[Dict[str, Any]]]" = queue.Queue(maxsize=QUEUE_MAX_SIZE) _worker_lock = threading.Lock() _worker: Optional[threading.Thread] = None @@ -104,22 +85,18 @@ def register_from_config(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: """ if not isinstance(cfg, dict): return [] - from utils import env_var_enabled if env_var_enabled("HERMES_SAFE_MODE"): logger.info("HERMES_SAFE_MODE=1 — outbound webhook registration skipped") return [] - targets = iter_configured_targets(cfg) if not targets: return [] - from hermes_cli.plugins import get_plugin_manager manager = get_plugin_manager() home_key = _home_key() - registered: List[WebhookTarget] = [] with _registered_lock: for target in targets: @@ -128,19 +105,15 @@ def register_from_config(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: key = (home_key, event, target.url) if key in _registered: continue - manager._hooks.setdefault(event, []).append( - _make_callback(event, target) - ) + manager._hooks.setdefault(event, []).append(_make_callback(event, target)) _registered.add(key) wired_any = True logger.info( - "outbound webhook registered: %s -> %s (matcher=%s, " - "timeout=%ds)", + "outbound webhook registered: %s -> %s (matcher=%s, timeout=%ds)", event, target.label, target.matcher, target.timeout, ) if wired_any: registered.append(target) - return registered @@ -149,9 +122,14 @@ def iter_configured_targets(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget if not isinstance(cfg, dict): return [] hooks_cfg = cfg.get("hooks") - return _parse_outbound_block( - hooks_cfg.get("outbound") if isinstance(hooks_cfg, dict) else None - ) + raw = hooks_cfg.get("outbound") if isinstance(hooks_cfg, dict) else None + if raw is None: + return [] + if not isinstance(raw, list): + logger.warning("hooks.outbound must be a list of webhook targets; got %s", type(raw).__name__) + return [] + targets = (_parse_single_target(i, entry) for i, entry in enumerate(raw)) + return [t for t in targets if t is not None] def flush(timeout: float = 5.0) -> bool: @@ -167,11 +145,9 @@ def flush(timeout: float = 5.0) -> bool: def re_register_config_hooks() -> None: - """Re-register outbound webhooks after a plugin force-reload cleared ``_hooks``. - - Only the current home's idempotence keys are cleared so a force-reload in one - profile cannot invalidate another profile's still-live registration. - """ + """Re-register outbound webhooks after a plugin force-reload cleared ``_hooks``. Only the + current home's idempotence keys are cleared so a force-reload in one profile cannot + invalidate another profile's still-live registration.""" from hermes_cli.config import load_config _forget_home_registrations(_registered, _registered_lock) @@ -192,19 +168,6 @@ def reset_for_tests() -> None: # --- Config parsing ------------------------------------------------------------- -def _parse_outbound_block(raw: Any) -> List[WebhookTarget]: - if raw is None: - return [] - if not isinstance(raw, list): - logger.warning( - "hooks.outbound must be a list of webhook targets; got %s", - type(raw).__name__, - ) - return [] - targets = (_parse_single_target(i, entry) for i, entry in enumerate(raw)) - return [t for t in targets if t is not None] - - def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: from hermes_cli.plugins import VALID_HOOKS @@ -255,7 +218,6 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: except (TypeError, ValueError): warn(".timeout must be an int (got %r); using default %ds", timeout_raw, DEFAULT_TIMEOUT_SECONDS) timeout = DEFAULT_TIMEOUT_SECONDS - timeout = max(1, min(timeout, MAX_TIMEOUT_SECONDS)) name = raw.get("name") return WebhookTarget( @@ -264,7 +226,7 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: name=name.strip() if isinstance(name, str) else "", secret=_resolve_secret(index, raw), matcher=matcher, - timeout=timeout, + timeout=max(1, min(timeout, MAX_TIMEOUT_SECONDS)), ) @@ -276,8 +238,8 @@ def _resolve_secret(index: int, raw: Dict[str, Any]) -> Optional[str]: if value: return value logger.warning( - "hooks.outbound[%d].secret_env=%r is not set in the environment " - "— deliveries will be UNSIGNED", index, secret_env.strip(), + "hooks.outbound[%d].secret_env=%r is not set in the environment — deliveries will be UNSIGNED", + index, secret_env.strip(), ) return None secret = raw.get("secret") @@ -297,8 +259,7 @@ def _make_callback(event: str, target: WebhookTarget): body = _serialize_payload(event, kwargs, delivery_id) except Exception: # a bad payload must not hurt the loop logger.warning( - "outbound webhook payload serialization failed (event=%s " - "target=%s)", event, target.label, exc_info=True, + "outbound webhook payload serialization failed (event=%s target=%s)", event, target.label, exc_info=True, ) return None _enqueue(_build_delivery(event, target, body, delivery_id)) @@ -309,14 +270,10 @@ def _make_callback(event: str, target: WebhookTarget): return _callback -def _serialize_payload( - event: str, kwargs: Dict[str, Any], delivery_id: str, -) -> bytes: - """Render the POST body: shell-hooks stdin shape plus delivery metadata. - - ``delivery_id`` (also the ``X-Hermes-Delivery`` header) and ``timestamp`` live - inside the HMAC-signed body, so they double as replay protection. - """ +def _serialize_payload(event: str, kwargs: Dict[str, Any], delivery_id: str) -> bytes: + """Render the POST body: shell-hooks stdin shape plus delivery metadata. ``delivery_id`` + (also the ``X-Hermes-Delivery`` header) and ``timestamp`` live inside the HMAC-signed + body, so they double as replay protection.""" # Profile resolved at fire time so a multiplexed gateway's receivers can tell which profile emitted. from hermes_cli.profiles import get_active_profile_name @@ -330,9 +287,7 @@ def _serialize_payload( return json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8") -def _build_delivery( - event: str, target: WebhookTarget, body: bytes, delivery_id: str, -) -> Dict[str, Any]: +def _build_delivery(event: str, target: WebhookTarget, body: bytes, delivery_id: str) -> Dict[str, Any]: headers = { "Content-Type": "application/json", "User-Agent": "Hermes-Agent-Outbound-Webhook", @@ -340,17 +295,11 @@ def _build_delivery( "X-Hermes-Delivery": delivery_id, } if target.secret: - digest = hmac.new( - target.secret.encode("utf-8"), body, hashlib.sha256 - ).hexdigest() + digest = hmac.new(target.secret.encode("utf-8"), body, hashlib.sha256).hexdigest() headers["X-Hermes-Signature-256"] = f"sha256={digest}" return { - "url": target.url, - "label": target.label, - "event": event, - "body": body, - "headers": headers, - "timeout": target.timeout, + "url": target.url, "label": target.label, "event": event, + "body": body, "headers": headers, "timeout": target.timeout, } @@ -360,8 +309,8 @@ def _enqueue(delivery: Dict[str, Any]) -> None: _delivery_queue.put_nowait(delivery) except queue.Full: logger.warning( - "outbound webhook queue full (%d pending) — dropping %s event " - "for %s", QUEUE_MAX_SIZE, delivery["event"], delivery["label"], + "outbound webhook queue full (%d pending) — dropping %s event for %s", + QUEUE_MAX_SIZE, delivery["event"], delivery["label"], ) @@ -372,13 +321,10 @@ def _ensure_worker() -> None: with _worker_lock: if _worker is not None and _worker.is_alive(): return - _worker = threading.Thread( - target=_worker_loop, name="outbound-webhooks", daemon=True, - ) + _worker = threading.Thread(target=_worker_loop, name="outbound-webhooks", daemon=True) _worker.start() - # Daemon worker: a short-lived process could exit right after enqueuing - # on_session_end. Drain at interpreter shutdown, bounded so a dead - # endpoint can only delay exit, never hang it. + # Daemon worker: a short-lived process could exit right after enqueuing on_session_end. + # Drain at interpreter shutdown, bounded so a dead endpoint can only delay exit, never hang it. atexit.register(flush, timeout=5.0) @@ -411,49 +357,34 @@ _opener = urlrequest.build_opener(_NoRedirectHandler) def _deliver(delivery: Dict[str, Any]) -> None: """POST with bounded retries: retry on connection errors and 5xx; 4xx and 3xx are final.""" + event, label = delivery["event"], delivery["label"] last_error = "" for attempt in range(1, MAX_DELIVERY_ATTEMPTS + 1): - req = urlrequest.Request( - delivery["url"], - data=delivery["body"], - headers=delivery["headers"], - method="POST", - ) + req = urlrequest.Request(delivery["url"], data=delivery["body"], headers=delivery["headers"], method="POST") try: with _opener.open(req, timeout=delivery["timeout"]) as resp: status = getattr(resp, "status", 200) if 200 <= status < 300: - logger.debug( - "outbound webhook delivered: %s -> %s (HTTP %d)", - delivery["event"], delivery["label"], status, - ) + logger.debug("outbound webhook delivered: %s -> %s (HTTP %d)", event, label, status) return last_error = f"HTTP {status}" except urlerror.HTTPError as exc: last_error = f"HTTP {exc.code}" if 300 <= exc.code < 400: logger.warning( - "outbound webhook target redirected (event=%s target=%s): " - "%s -> %s — redirects are not followed; update the " - "configured url", delivery["event"], delivery["label"], - last_error, exc.headers.get("Location", "?"), + "outbound webhook target redirected (event=%s target=%s): %s -> %s — redirects are not " + "followed; update the configured url", event, label, last_error, exc.headers.get("Location", "?"), ) return if 400 <= exc.code < 500: - logger.warning( - "outbound webhook rejected (event=%s target=%s): %s — " - "not retrying", delivery["event"], delivery["label"], - last_error, - ) + logger.warning("outbound webhook rejected (event=%s target=%s): %s — not retrying", event, label, last_error) return except Exception as exc: last_error = str(exc) or type(exc).__name__ - if attempt < MAX_DELIVERY_ATTEMPTS: time.sleep(RETRY_BACKOFF_SECONDS * attempt) logger.warning( - "outbound webhook delivery failed after %d attempt(s) (event=%s " - "target=%s): %s", - MAX_DELIVERY_ATTEMPTS, delivery["event"], delivery["label"], last_error, + "outbound webhook delivery failed after %d attempt(s) (event=%s target=%s): %s", + MAX_DELIVERY_ATTEMPTS, event, label, last_error, ) From 052582de2d7cb3573ff4ece883347eec04eeb5d7 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:32:02 -0700 Subject: [PATCH 3/9] =?UTF-8?q?refactor(agent):=20compact=20monitoring/*?= =?UTF-8?q?=20and=20lsp/workspace=20=E2=80=94=20contextlib.suppress,=20sha?= =?UTF-8?q?red=20platform=20error-code=20helper,=20nearest=5Froot=20marker?= =?UTF-8?q?=20probe,=20docstring=20compaction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/lsp/workspace.py | 98 ++++++++------------ agent/monitoring/__init__.py | 7 +- agent/monitoring/cron_health.py | 43 ++------- agent/monitoring/emitter.py | 28 ++---- agent/monitoring/events.py | 6 +- agent/monitoring/gateway_health.py | 73 +++++---------- agent/monitoring/gateway_health_export.py | 106 ++++++--------------- agent/monitoring/otlp_exporter.py | 107 ++++++---------------- agent/monitoring/policy.py | 4 +- agent/monitoring/redaction.py | 29 ++---- 10 files changed, 144 insertions(+), 357 deletions(-) diff --git a/agent/lsp/workspace.py b/agent/lsp/workspace.py index a33e7908c6..5b220fcab7 100644 --- a/agent/lsp/workspace.py +++ b/agent/lsp/workspace.py @@ -1,11 +1,10 @@ """Workspace and project-root resolution for LSP. -1. **Workspace gate** — LSP only runs when the cwd (or the edited file) sits - inside a git worktree. Files outside any git root never trigger LSP, - which keeps gateway users on user-home cwd's from spawning daemons. -2. **nearest_root** — the per-server project-root walk: up from a start path - looking for marker files (``pyproject.toml``, ``Cargo.toml``, ...), - optionally bailing if an exclude marker shows up first. +1. **Workspace gate** — LSP only runs when the cwd (or the edited file) sits inside a git + worktree, so gateway users on user-home cwd's never spawn daemons. +2. **nearest_root** — the per-server project-root walk: up from a start path looking for marker + files (``pyproject.toml``, ``Cargo.toml``, ...), optionally bailing if an exclude marker + shows up first. """ from __future__ import annotations @@ -16,8 +15,7 @@ from typing import Iterable, Iterator, Optional, Tuple logger = logging.getLogger("agent.lsp.workspace") -# Cache: start dir → (worktree_root, is_git) so repeated calls don't re-stat. -# Cleared on shutdown. +# Cache: start dir → (worktree_root, is_git) so repeated calls don't re-stat. Cleared on shutdown. _workspace_cache: dict = {} # Walk cap: the deepest reasonable monorepo is well under 64 levels; bounds a @@ -26,11 +24,8 @@ _MAX_WALK = 64 def normalize_path(path: str) -> str: - """Expand ``~``, make absolute, collapse ``.``/``..``. - - Symlinks are deliberately NOT resolved — some servers (rust-analyzer's - Cargo workspace identity) care, and we want the path the user typed. - """ + """Expand ``~``, make absolute, collapse ``.``/``..``. Symlinks are deliberately NOT resolved — + some servers (rust-analyzer's Cargo workspace identity) care, and we want the path the user typed.""" return os.path.abspath(os.path.expanduser(path)) @@ -62,11 +57,9 @@ def find_git_worktree(start: str) -> Optional[str]: start_path = _start_dir(start) if start_path is None: return None - cached = _workspace_cache.get(str(start_path)) if cached is not None: return cached[0] - for cur in _walk_up(start_path): try: if (cur / ".git").exists(): @@ -74,30 +67,24 @@ def find_git_worktree(start: str) -> Optional[str]: _workspace_cache[str(start_path)] = (resolved, True) return resolved except OSError: - # Permission error on a parent dir — bail out cleanly. - break - + break # permission error on a parent dir — bail out cleanly _workspace_cache[str(start_path)] = (None, False) return None def is_inside_workspace(path: str, workspace_root: str) -> bool: - """True iff ``path`` is inside (or equal to) ``workspace_root``. - - Symlinks are not resolved: a symlink pointing outside still counts as - outside, matching servers that reject didOpen for unrelated files. - """ + """True iff ``path`` is inside (or equal to) ``workspace_root``. Symlinks are not resolved: a + symlink pointing outside still counts as outside, matching servers that reject didOpen for + unrelated files.""" p = normalize_path(path) root = normalize_path(workspace_root) if p == root: return True # commonpath handles case-insensitive filesystems on macOS/Windows. try: - common = os.path.commonpath([p, root]) + return os.path.commonpath([p, root]) == root except ValueError: - # Different drives on Windows. - return False - return common == root + return False # different drives on Windows def nearest_root( @@ -109,51 +96,42 @@ def nearest_root( ) -> Optional[str]: """Walk up from ``start`` for the directory containing the first matched marker. - Returns ``None`` past ``ceiling`` (or the filesystem root), or when an - exclude marker is found first — the server is gated off for that file - (e.g. typescript skips deno projects when ``deno.json`` precedes - ``package.json``). Marker names are exact filenames — no globs. + Returns ``None`` past ``ceiling`` (or the filesystem root), or when an exclude marker is found + first — the server is gated off for that file (e.g. typescript skips deno projects when + ``deno.json`` precedes ``package.json``). Marker names are exact filenames — no globs. """ start_path = _start_dir(start) if start_path is None: return None ceiling_path = Path(normalize_path(ceiling)) if ceiling else None - markers_list = list(markers) excludes_list = list(excludes) if excludes else [] + def present(cur: Path, names: list) -> bool: + for name in names: + try: + if (cur / name).exists(): + return True + except OSError: + continue + return False + for cur in _walk_up(start_path): # Excludes are checked before markers at each level. - for exc in excludes_list: - try: - if (cur / exc).exists(): - return None - except OSError: - continue - for marker in markers_list: - try: - if (cur / marker).exists(): - return str(cur) - except OSError: - continue + if present(cur, excludes_list): + return None + if present(cur, markers_list): + return str(cur) if ceiling_path is not None and cur == ceiling_path: return None return None -def resolve_workspace_for_file( - file_path: str, - *, - cwd: Optional[str] = None, -) -> Tuple[Optional[str], bool]: - """Return ``(workspace_root, gated_in)`` for a file. - - The cwd's worktree wins when the file is inside it; otherwise the file's - own worktree is the fallback anchor (monorepos / unrelated checkouts). - ``(None, False)`` when neither is in a git worktree. - """ - cwd = cwd or os.getcwd() - cwd_root = find_git_worktree(cwd) +def resolve_workspace_for_file(file_path: str, *, cwd: Optional[str] = None) -> Tuple[Optional[str], bool]: + """Return ``(workspace_root, gated_in)`` for a file. The cwd's worktree wins when the file is + inside it; otherwise the file's own worktree is the fallback anchor (monorepos / unrelated + checkouts). ``(None, False)`` when neither is in a git worktree.""" + cwd_root = find_git_worktree(cwd or os.getcwd()) if cwd_root is not None and is_inside_workspace(file_path, cwd_root): return cwd_root, True file_root = find_git_worktree(file_path) @@ -168,10 +146,6 @@ def clear_cache() -> None: __all__ = [ - "find_git_worktree", - "is_inside_workspace", - "nearest_root", - "normalize_path", - "resolve_workspace_for_file", + "find_git_worktree", "is_inside_workspace", "nearest_root", "normalize_path", "resolve_workspace_for_file", "clear_cache", ] diff --git a/agent/monitoring/__init__.py b/agent/monitoring/__init__.py index 20b4dc40f4..85f95932b9 100644 --- a/agent/monitoring/__init__.py +++ b/agent/monitoring/__init__.py @@ -14,9 +14,4 @@ from . import emitter, events emit = emitter.emit get_emitter = emitter.get_emitter -__all__ = [ - "emitter", - "events", - "emit", - "get_emitter", -] +__all__ = ["emitter", "events", "emit", "get_emitter"] diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py index 04e5122f76..40a2344455 100644 --- a/agent/monitoring/cron_health.py +++ b/agent/monitoring/cron_health.py @@ -9,11 +9,7 @@ from datetime import datetime from typing import Any, Callable, Optional from agent.monitoring.events import CronExecutionEvent -from agent.monitoring.gateway_health import ( - GatewayMetric, - _contains_any, - _safe_instance_id, -) +from agent.monitoring.gateway_health import GatewayMetric, _contains_any, _safe_instance_id from cron.jobs import ( _compute_grace_seconds, get_catch_up_occurrence_count, @@ -27,9 +23,7 @@ from hermes_time import now as _now logger = logging.getLogger(__name__) _KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"} _KNOWN_SOURCES = {"builtin", "direct", "external"} -_KNOWN_DELIVERY_OUTCOMES = { - "delivered", "failed", "suppressed", "suppressed_acked", "not_configured", -} +_KNOWN_DELIVERY_OUTCOMES = {"delivered", "failed", "suppressed", "suppressed_acked", "not_configured"} _TERMINAL_STATUSES = {"completed", "failed", "unknown"} @@ -47,7 +41,6 @@ _AUTH_RE = re.compile( r"|unauthorized|forbidden|bearer|401|403)\b|\b(?:access|api|refresh) token\b" ) - # Ordered (predicate, class) rules; first match wins. Auth uses word boundaries so # "oauth"/"tokenizer"/"HTTP 4015" do not false-positive. _CRON_ERROR_RULES: tuple[tuple[Callable[[str], bool], str], ...] = ( @@ -86,9 +79,7 @@ def _duration_ms(record: dict[str, Any]) -> Optional[int]: return max(0, duration) -def project_execution_event( - record: dict[str, Any], *, delivery_outcome: Optional[str] = None -) -> CronExecutionEvent: +def project_execution_event(record: dict[str, Any], *, delivery_outcome: Optional[str] = None) -> CronExecutionEvent: status = str(record.get("status") or "unknown").lower() source = str(record.get("source") or "unknown").lower() outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None @@ -98,20 +89,12 @@ def project_execution_event( # Unknown non-empty sources are bucketed as "external" (not dropped to unknown). source=source if source in _KNOWN_SOURCES or source == "unknown" else "external", duration_ms=_duration_ms(record), - delivery_outcome=( - outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None - ), - error_class=( - classify_cron_error(record.get("error")) - if status in {"failed", "unknown"} - else None - ), + delivery_outcome=outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None, + error_class=classify_cron_error(record.get("error")) if status in {"failed", "unknown"} else None, ) -def emit_execution_state( - record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None -) -> None: +def emit_execution_state(record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None) -> None: """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier.""" if not record: return @@ -137,8 +120,7 @@ def _is_overdue(job: dict[str, Any], now: datetime) -> bool: try: if next_run.tzinfo is None and now.tzinfo is not None: next_run = next_run.replace(tzinfo=now.tzinfo) - lateness = (now - next_run).total_seconds() - return lateness > _compute_grace_seconds(schedule) + return (now - next_run).total_seconds() > _compute_grace_seconds(schedule) except (TypeError, ValueError): return False @@ -146,11 +128,7 @@ def _is_overdue(job: dict[str, Any], now: datetime) -> bool: def _job_metrics(metrics: list[GatewayMetric]) -> None: enabled = [job for job in load_jobs() if job.get("enabled", True)] metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {})) - metrics.append(GatewayMetric( - "hermes.cron.jobs.overdue", - sum(1 for job in enabled if _is_overdue(job, _now())), - {}, - )) + metrics.append(GatewayMetric("hermes.cron.jobs.overdue", sum(1 for job in enabled if _is_overdue(job, _now())), {})) def _freshness_metric(name: str, reader: Callable[[], Optional[float]]) -> Callable[[list[GatewayMetric]], None]: @@ -191,9 +169,6 @@ def build_cron_health_snapshot() -> CronHealthSnapshot: __all__ = [ - "CronHealthSnapshot", - "build_cron_health_snapshot", - "classify_cron_error", - "emit_execution_state", + "CronHealthSnapshot", "build_cron_health_snapshot", "classify_cron_error", "emit_execution_state", "project_execution_event", ] diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py index d63ebb73ac..2858d257c6 100644 --- a/agent/monitoring/emitter.py +++ b/agent/monitoring/emitter.py @@ -14,6 +14,7 @@ import logging import queue import threading import time +from contextlib import suppress from typing import Any, Dict, Optional logger = logging.getLogger(__name__) @@ -67,9 +68,7 @@ class MonitoringEmitter: with self._lock: if self._started: return - self._thread = threading.Thread( - target=self._run, name="hermes-monitoring-dispatch", daemon=True - ) + self._thread = threading.Thread(target=self._run, name="hermes-monitoring-dispatch", daemon=True) self._thread.start() self._started = True @@ -106,10 +105,8 @@ class MonitoringEmitter: self._enabled = True def unsubscribe(self, callback) -> None: - try: + with suppress(ValueError): self._subscribers.remove(callback) - except ValueError: - pass if not self._subscribers: self._enabled = False @@ -118,19 +115,13 @@ class MonitoringEmitter: """Wait boundedly for queued and in-flight batches to finish dispatch.""" if timeout <= 0: return - finished = threading.Event() def _wait_for_completion() -> None: self._q.join() finished.set() - waiter = threading.Thread( - target=_wait_for_completion, - name="hermes-monitoring-flush", - daemon=True, - ) - waiter.start() + threading.Thread(target=_wait_for_completion, name="hermes-monitoring-flush", daemon=True).start() finished.wait(timeout=timeout) def stats(self) -> Dict[str, int]: @@ -175,16 +166,9 @@ def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None global _EMITTER with _EMITTER_LOCK: if _EMITTER is not None and emitter is not _EMITTER: - try: + with suppress(Exception): _EMITTER.close() - except Exception: - pass _EMITTER = emitter -__all__ = [ - "MonitoringEmitter", - "get_emitter", - "emit", - "reset_emitter_for_tests", -] +__all__ = ["MonitoringEmitter", "get_emitter", "emit", "reset_emitter_for_tests"] diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py index ee51d7ba28..a9bb57dcb5 100644 --- a/agent/monitoring/events.py +++ b/agent/monitoring/events.py @@ -81,8 +81,4 @@ class CronExecutionEvent(_MonitoringEvent): ts_ns: int = field(default_factory=time.time_ns) -__all__ = [ - "GatewayHealthEvent", - "GatewayDiagnosticEvent", - "CronExecutionEvent", -] +__all__ = ["GatewayHealthEvent", "GatewayDiagnosticEvent", "CronExecutionEvent"] diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index f3ac45f8f3..ea7ca161ac 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -75,9 +75,7 @@ def classify_gateway_error(raw: Any) -> str: return next((label for match, label in _GATEWAY_ERROR_RULES if match(s)), "unknown") -def classify_exit_reason( - raw: Any, *, state: Any, restart_requested: bool -) -> Optional[str]: +def classify_exit_reason(raw: Any, *, state: Any, restart_requested: bool) -> Optional[str]: """Reduce free-form shutdown text to a bounded operational class.""" if restart_requested: return "restart_requested" @@ -142,9 +140,7 @@ def _parse_active_agents(raw: Any) -> int: def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: try: from gateway.status import derive_gateway_busy - return derive_gateway_busy( - gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents - ) + return derive_gateway_busy(gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents) except Exception: return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0) @@ -179,6 +175,11 @@ def _platforms_of(runtime: Optional[dict[str, Any]]) -> dict[str, Any]: return raw if isinstance(raw, dict) else {} +def _platform_error_code(pdata: dict[str, Any]) -> str: + # classify_* is idempotent on its own labels, so error_class == error_code downstream. + return classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) + + def build_gateway_health_snapshot( runtime: Optional[dict[str, Any]], *, @@ -211,8 +212,7 @@ def build_gateway_health_snapshot( for platform, pdata in platforms.items(): pdata = pdata if isinstance(pdata, dict) else {} state = _bounded_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES) - # classify_* is idempotent on its own labels, so error_class == error_code here. - error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) + error_code = _platform_error_code(pdata) is_degraded = state in _FATAL_PLATFORM_STATES if is_degraded: fatal_count += 1 @@ -223,13 +223,8 @@ def build_gateway_health_snapshot( )) if is_degraded: events.append(GatewayDiagnosticEvent( - name="platform.fatal", - subsystem=f"platform.{platform}", - platform=str(platform), - error_code=error_code, - error_class=error_code, - profile=profile, - version=version, + name="platform.fatal", subsystem=f"platform.{platform}", platform=str(platform), + error_code=error_code, error_class=error_code, profile=profile, version=version, severity="error" if state == "fatal" else "warning", )) @@ -282,9 +277,7 @@ def _lifecycle_events( gateway_state=new_state, old_state=old_state, new_state=new_state, - exit_reason=classify_exit_reason( - current.get("exit_reason"), state=new_state, restart_requested=restart_requested - ), + exit_reason=classify_exit_reason(current.get("exit_reason"), state=new_state, restart_requested=restart_requested), restart_requested=restart_requested, active_agents=_parse_active_agents(current.get("active_agents", 0)), profile=profile, @@ -296,13 +289,8 @@ def _lifecycle_events( if new_state == "startup_failed": error_class = classify_gateway_error(current.get("exit_reason") or "startup_failed") out.append(GatewayDiagnosticEvent( - name="gateway.startup_failed", - subsystem="gateway", - error_class=error_class, - error_code=error_class, - profile=profile, - version=version, - severity="error", + name="gateway.startup_failed", subsystem="gateway", error_class=error_class, error_code=error_class, + profile=profile, version=version, severity="error", )) if new_state == "stopped": out.append(health("gateway.exit")) @@ -323,30 +311,20 @@ def _platform_events( new_state = _optional_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES) if old_state == new_state or not new_state: continue - error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) + error_code = _platform_error_code(pdata) common: dict[str, Any] = dict( - subsystem=f"platform.{platform}", - platform=str(platform), - error_code=error_code, - error_class=error_code, - profile=profile, - version=version, - severity="error" if new_state in {"fatal", "failed", "error"} else "warning", + subsystem=f"platform.{platform}", platform=str(platform), error_code=error_code, error_class=error_code, + profile=profile, version=version, severity="error" if new_state in {"fatal", "failed", "error"} else "warning", ) - out.append(GatewayDiagnosticEvent( - name="platform.state_change", old_state=old_state, new_state=new_state, **common - )) + out.append(GatewayDiagnosticEvent(name="platform.state_change", old_state=old_state, new_state=new_state, **common)) if new_state in _FATAL_PLATFORM_STATES: out.append(GatewayDiagnosticEvent(name="platform.fatal", **common)) return out def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None: - """Emit immediate content-free gateway events for runtime status changes. - - Called by gateway.status.write_runtime_status after persisting the new status. - Fully fail-open: failures never affect gateway status writes. - """ + """Emit immediate content-free gateway events for runtime status changes. Called by + gateway.status.write_runtime_status after persisting; fully fail-open.""" try: ctx = dict(profile=_safe_profile(), version=_safe_version()) for ev in _lifecycle_events(previous, current, **ctx) + _platform_events(previous, current, **ctx): @@ -379,7 +357,7 @@ class GatewayDiagnosticLogHandler(logging.Handler): return subsystem = subsystem_for_logger(record.name) error_class = classify_gateway_error(record.getMessage()) - event = GatewayDiagnosticEvent( + emitter.get_emitter().emit(GatewayDiagnosticEvent( name=f"gateway.log.{record.levelname.lower()}", subsystem=subsystem, source_logger=source_logger_for_export(record.name), @@ -389,17 +367,12 @@ class GatewayDiagnosticLogHandler(logging.Handler): profile=self.profile, version=self.version, severity=record.levelname.lower(), - ) - emitter.get_emitter().emit(event) + )) except Exception: logger.debug("gateway diagnostic emit failed", exc_info=True) __all__ = [ - "GatewayMetric", - "GatewayHealthSnapshot", - "GatewayDiagnosticLogHandler", - "build_gateway_health_snapshot", - "classify_gateway_error", - "source_logger_for_export", + "GatewayMetric", "GatewayHealthSnapshot", "GatewayDiagnosticLogHandler", + "build_gateway_health_snapshot", "classify_gateway_error", "source_logger_for_export", ] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index fa7d1daee1..81f4bfd95d 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -1,9 +1,8 @@ """Gateway Health & Diagnostics OTLP export runtime. -Emits operator-owned gateway service-health metrics plus narrow redacted -diagnostic events. Deliberately in-process and fail-open so it works under -systemd, launchd, s6, containers, tmux, nohup, or a plain shell without a -sidecar/watchdog dependency. +Emits operator-owned gateway service-health metrics plus narrow redacted diagnostic events. +Deliberately in-process and fail-open so it works under systemd, launchd, s6, containers, +tmux, nohup, or a plain shell without a sidecar/watchdog dependency. """ from __future__ import annotations @@ -12,6 +11,7 @@ import importlib import logging import os import threading +from contextlib import suppress from dataclasses import dataclass from typing import Any, Callable, Dict, Optional @@ -47,21 +47,11 @@ _METRICS_SDK = ( ) # Every gauge the runtime snapshot can emit MUST be listed here or it is silently dropped. _OBSERVABLE_METRIC_NAMES = ( - "hermes.gateway.up", - "hermes.gateway.state", - "hermes.gateway.active_agents", - "hermes.gateway.busy", - "hermes.gateway.drainable", - "hermes.gateway.restart_requested", - "hermes.gateway.background_work", - "hermes.gateway.background_delegations", - "hermes.platform.up", - "hermes.platform.degraded", - "hermes.cron.scheduler.heartbeat_age_seconds", - "hermes.cron.scheduler.last_success_age_seconds", - "hermes.cron.scheduler.catch_up_occurrences", - "hermes.cron.jobs.enabled", - "hermes.cron.jobs.running", + "hermes.gateway.up", "hermes.gateway.state", "hermes.gateway.active_agents", "hermes.gateway.busy", + "hermes.gateway.drainable", "hermes.gateway.restart_requested", "hermes.gateway.background_work", + "hermes.gateway.background_delegations", "hermes.platform.up", "hermes.platform.degraded", + "hermes.cron.scheduler.heartbeat_age_seconds", "hermes.cron.scheduler.last_success_age_seconds", + "hermes.cron.scheduler.catch_up_occurrences", "hermes.cron.jobs.enabled", "hermes.cron.jobs.running", "hermes.cron.jobs.overdue", ) @@ -88,21 +78,17 @@ class GatewayHealthExportRuntime: if self.thread is not None: self.thread.join(timeout=0.25) if self.log_handler is not None: - try: + with suppress(Exception): logging.getLogger().removeHandler(self.log_handler) - except Exception: - pass # Producers are stopped; drain queued/in-flight events BEFORE detaching subscribers # so the terminal lifecycle event cannot race exporter shutdown. Bounded, fail-open. subscribers = [item for item in (self.streamer, self.log_streamer) if item is not None] - try: + with suppress(Exception): bus = emitter.get_emitter() bus.flush(timeout=1.0) for sub in subscribers: bus.unsubscribe(sub) - except Exception: - pass # Network flush/close runs under one bounded daemon-thread deadline so it can # never delay gateway teardown indefinitely. @@ -110,15 +96,11 @@ class GatewayHealthExportRuntime: def _close() -> None: for item in closeables: - try: + with suppress(Exception): item.shutdown() - except Exception: - pass if closeables: - worker = threading.Thread( - target=_close, name="hermes-gateway-health-export-shutdown", daemon=True - ) + worker = threading.Thread(target=_close, name="hermes-gateway-health-export-shutdown", daemon=True) worker.start() worker.join(timeout=2.0) @@ -172,11 +154,7 @@ def _read_gateway_snapshot(config: Dict[str, Any]): except Exception: runtime = {} return build_gateway_health_snapshot( - runtime, - gateway_running=True, - profile=_profile(), - install_id=_install_id(config), - version=_version(), + runtime, gateway_running=True, profile=_profile(), install_id=_install_id(config), version=_version(), supervision_mode=_supervision_mode(), ) @@ -197,27 +175,22 @@ def _count(failure_msg: str, module: str, read: Callable[[Any], Any]) -> int: def _read_background_work_count() -> int: - """Live background/subagent work that ``active_agents`` deliberately does NOT include. - - ``active_agents`` counts foreground turns + in-flight cron + API runs; backgrounded - ``delegate_task`` subagents, ``terminal(background=true)`` processes and kanban workers - are tracked only by the scale-to-zero guard, so without this a peer churning through - subagents shows ``active_agents=0``. TASK-granular: a fan-out batch of N contributes N - (real concurrent load), unlike the pool's one-slot-per-batch accounting. Content-free. - """ + """Live background/subagent work that ``active_agents`` deliberately does NOT include + (``active_agents`` = foreground turns + in-flight cron + API runs; backgrounded + ``delegate_task`` subagents, ``terminal(background=true)`` processes and kanban workers are + tracked only by the scale-to-zero guard). TASK-granular: a fan-out batch of N contributes N + (real concurrent load), unlike the pool's one-slot-per-batch accounting. Content-free.""" return ( - _count("background-work async-delegation count failed", "tools.async_delegation", - lambda m: m.active_task_count()) + _count("background-work async-delegation count failed", "tools.async_delegation", lambda m: m.active_task_count()) + _count("background-work process-registry count failed", "tools.process_registry", lambda m: m.process_registry.count_running()) ) def _read_background_delegations_count() -> int: - """Live async delegation UNITS (dispatch/pool slots): a batch counts ONE regardless of - fan-out width, matching the pool's capacity accounting — so operators can see slot - pressure (alert vs ``max_concurrent_children``) alongside ``background_work``'s real load. - Delegations only; terminal/kanban work is already folded into ``background_work``.""" + """Live async delegation UNITS (dispatch/pool slots): a batch counts ONE regardless of fan-out + width, matching the pool's capacity accounting — slot pressure (alert vs + ``max_concurrent_children``) alongside ``background_work``'s real load. Delegations only.""" return _count("background-delegations count failed", "tools.async_delegation", lambda m: m.active_count()) @@ -233,20 +206,14 @@ def _read_runtime_snapshot(config: Dict[str, Any]): ): gateway_snapshot.metrics.append(GatewayMetric(name=name, value=read(), attributes=base)) except Exception as exc: - logger.warning( - "background-work snapshot unavailable; metric not exported (error_type=%s)", - type(exc).__name__, - ) + logger.warning("background-work snapshot unavailable; metric not exported (error_type=%s)", type(exc).__name__) logger.debug("background-work snapshot traceback", exc_info=True) try: cron_snapshot = _read_cron_snapshot() except Exception as exc: # Cron telemetry silently dropping out is a release-relevant regression: WARN with only # the exception *type* (the message could carry paths); exc_info stays on DEBUG. - logger.warning( - "cron health snapshot unavailable; cron telemetry not exported (error_type=%s)", - type(exc).__name__, - ) + logger.warning("cron health snapshot unavailable; cron telemetry not exported (error_type=%s)", type(exc).__name__) logger.debug("cron health snapshot traceback", exc_info=True) return gateway_snapshot gateway_snapshot.metrics.extend(cron_snapshot.metrics) @@ -268,9 +235,7 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: exporter = sdk["OTLPMetricExporter"](**_exporter_kwargs(config, "metrics")) interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000 reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms) - provider = sdk["MeterProvider"]( - metric_readers=[reader], resource=_resource(config, sdk, "gateway_health") - ) + provider = sdk["MeterProvider"](metric_readers=[reader], resource=_resource(config, sdk, "gateway_health")) meter = provider.get_meter("hermes.gateway.health") Observation = sdk["Observation"] @@ -290,12 +255,7 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: _SEVERITY_NAMES = { - "critical": "FATAL", - "fatal": "FATAL", - "error": "ERROR", - "info": "INFO", - "information": "INFO", - "debug": "DEBUG", + "critical": "FATAL", "fatal": "FATAL", "error": "ERROR", "info": "INFO", "information": "INFO", "debug": "DEBUG", } @@ -309,9 +269,7 @@ class GatewayDiagnosticLogStreamer(EmitterStreamer): def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]): self._provider = sdk["LoggerProvider"](resource=_resource(config, sdk, "gateway_diagnostics")) - self._processor = sdk["BatchLogRecordProcessor"]( - sdk["OTLPLogExporter"](**_exporter_kwargs(config, "logs")) - ) + self._processor = sdk["BatchLogRecordProcessor"](sdk["OTLPLogExporter"](**_exporter_kwargs(config, "logs"))) self._provider.add_log_record_processor(self._processor) self._logger = self._provider.get_logger(_DEFAULT_DIAGNOSTIC_SCOPE) self._sdk = sdk @@ -382,8 +340,7 @@ def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRu sdk = _require_metrics_sdk(prompt=False) except Exception: logger.warning( - "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; " - "install 'hermes-agent[otlp]'", + "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; install 'hermes-agent[otlp]'", exc_info=True, ) return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable") @@ -423,7 +380,4 @@ def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRu return runtime -__all__ = [ - "GatewayHealthExportRuntime", - "start_gateway_health_export", -] +__all__ = ["GatewayHealthExportRuntime", "start_gateway_health_export"] diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index f1c12fa4dc..0c2d8e102c 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -1,18 +1,12 @@ """Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP. -Maps gateway monitoring events to OTel spans and sends them to the operator -configured ``monitoring.export.otlp`` endpoint (no default destination ships). -Also hosts the OTLP plumbing shared with ``gateway_health_export`` (SDK loading, -header resolution, resource attributes, endpoint mapping). - -* The OTel SDK is an optional extra (``pip install hermes-agent[otlp]``), - imported lazily so it is only required when export is actually used. -* ``headers_env`` maps a header name to an environment variable name; values - are read from the environment at export time and never logged or stored. -* The continuous subscriber runs in the emitter's dispatcher thread and is - fail-isolated, so an export error cannot affect the gateway. The - ``event_filter`` seam keeps future planes sharing the emitter from silently - riding along on this exporter. +Maps gateway monitoring events to OTel spans for the operator-configured +``monitoring.export.otlp`` endpoint (no default destination ships) and hosts the OTLP +plumbing shared with ``gateway_health_export`` (SDK loading, header resolution, resource +attributes, endpoint mapping). The OTel SDK is an optional extra (``hermes-agent[otlp]``) +imported lazily; ``headers_env`` values are read from the environment at export time and +never logged or stored. The continuous subscriber runs on the emitter's dispatcher thread, +fail-isolated, and ``event_filter`` keeps other planes from riding along on this exporter. """ from __future__ import annotations @@ -21,6 +15,7 @@ import importlib import logging import os import re +from contextlib import suppress from typing import Any, Callable, Dict, Iterable, List, Optional from agent.monitoring.gateway_health import _safe_instance_id @@ -56,27 +51,19 @@ _SDK_SYMBOLS: Dict[str, str] = { _SPAN_SDK = ("TracerProvider", "BatchSpanProcessor", "Resource", "OTLPSpanExporter", "SpanKind") -def _require_sdk( - names: Iterable[str] = _SPAN_SDK, *, auto_install: bool = True, prompt: bool = True -) -> Dict[str, Any]: +def _require_sdk(names: Iterable[str] = _SPAN_SDK, *, auto_install: bool = True, prompt: bool = True) -> Dict[str, Any]: """Import the named OTel SDK symbols, lazily installing the extra on first use. - Routes through tools.lazy_deps (feature 'export.otlp') — gated by - security.allow_lazy_installs and TTY-prompted (``prompt=False`` from - non-interactive contexts). Any lazy-install failure falls through to the - import attempt, which raises OTLPUnavailable with a manual install hint. + Routes through tools.lazy_deps (feature 'export.otlp') — gated by security.allow_lazy_installs + and TTY-prompted (``prompt=False`` from non-interactive contexts). Any lazy-install failure + falls through to the import attempt, which raises OTLPUnavailable with a manual install hint. """ if auto_install: - try: + with suppress(Exception): from tools.lazy_deps import ensure as _lazy_ensure _lazy_ensure("export.otlp", prompt=prompt) - except Exception: - pass try: - return { - name: getattr(importlib.import_module(_SDK_SYMBOLS[name]), name) - for name in names - } + return {name: getattr(importlib.import_module(_SDK_SYMBOLS[name]), name) for name in names} except Exception as e: # ImportError or partial install raise OTLPUnavailable( "OTLP export requires the optional dependency. Install with:\n" @@ -122,15 +109,8 @@ def _signal_endpoint(endpoint: str, signal: str) -> str: _RESOURCE_ATTRIBUTE_KEYS = frozenset({ - "service.name", - "service.namespace", - "service.version", - "service.instance.id", - "deployment.environment.name", - "cloud.provider", - "cloud.platform", - "cloud.region", - "telemetry.scope", + "service.name", "service.namespace", "service.version", "service.instance.id", + "deployment.environment.name", "cloud.provider", "cloud.platform", "cloud.region", "telemetry.scope", }) _SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$") @@ -158,12 +138,9 @@ def _safe_resource_attributes(raw: Any) -> Dict[str, str]: return attrs -def _runtime_resource_attributes( - config: Dict[str, Any], *, telemetry_scope: str -) -> Dict[str, str]: +def _runtime_resource_attributes(config: Dict[str, Any], *, telemetry_scope: str) -> Dict[str, str]: """Build the safe OTLP resource shared by spans, metrics and diagnostic logs.""" - gh = _monitoring_section(config, "gateway_health_export") - attrs = _safe_resource_attributes(gh.get("resource_attributes")) + attrs = _safe_resource_attributes(_monitoring_section(config, "gateway_health_export").get("resource_attributes")) attrs["service.name"] = "hermes-gateway" attrs["service.instance.id"] = _safe_instance_id(_install_id(config)) attrs["telemetry.scope"] = telemetry_scope @@ -177,8 +154,7 @@ def build_exporter(config: Dict[str, Any]): endpoint = otlp.get("endpoint") if not endpoint: raise ValueError("monitoring.export.otlp.endpoint is not set") - headers = _resolve_headers(otlp.get("headers_env")) - return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None) + return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=_resolve_headers(otlp.get("headers_env")) or None) def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]: @@ -187,8 +163,7 @@ def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]: def _make_provider(config: Dict[str, Any]): sdk = _require_sdk() - resource = sdk["Resource"].create(_resource_attributes(config)) - provider = sdk["TracerProvider"](resource=resource) + provider = sdk["TracerProvider"](resource=sdk["Resource"].create(_resource_attributes(config))) processor = sdk["BatchSpanProcessor"](build_exporter(config)) provider.add_span_processor(processor) return provider, processor @@ -234,9 +209,7 @@ def export_batch(provider, batch: List[Dict[str, Any]]) -> int: n = 0 for ev in batch: try: - name = f"hermes.{ev.get('event', 'event')}" - span = tracer.start_span(name, attributes=_span_attrs(ev)) - span.end() + tracer.start_span(f"hermes.{ev.get('event', 'event')}", attributes=_span_attrs(ev)).end() n += 1 except Exception: logger.debug("OTLP span map failed", exc_info=True) @@ -246,36 +219,25 @@ def export_batch(provider, batch: List[Dict[str, Any]]) -> int: # ── continuous streaming subscribers ───────────────────────────────────────── class EmitterStreamer: """Base for emitter subscribers owning an OTel provider + batch processor. - - Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter. - """ + Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.""" _provider: Any _processor: Any exported: int = 0 def shutdown(self) -> None: - try: + with suppress(Exception): from agent.monitoring.emitter import get_emitter get_emitter().unsubscribe(self) - except Exception: - pass - try: + with suppress(Exception): self._processor.force_flush() self._provider.shutdown() - except Exception: - pass class OTLPStreamer(EmitterStreamer): """A live subscriber that pushes each emitter batch to OTLP as spans.""" - def __init__( - self, - config: Dict[str, Any], - *, - event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, - ): + def __init__(self, config: Dict[str, Any], *, event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None): self._provider, self._processor = _make_provider(config) self._event_filter = event_filter self.exported = 0 @@ -303,16 +265,13 @@ def is_enabled(config: Dict[str, Any]) -> bool: def start_streaming( - config: Dict[str, Any], - *, - event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, + config: Dict[str, Any], *, event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, ) -> Optional[OTLPStreamer]: """If OTLP is enabled, attach a streamer to the singleton emitter. - ``event_filter`` scopes the exporter to its plane. Startup is non-interactive: - a configured-but-missing SDK is lazily installed once (prompt=False, gated by - security.allow_lazy_installs); if it still can't load, log and no-op — never - raise into startup. + ``event_filter`` scopes the exporter to its plane. Startup is non-interactive: a + configured-but-missing SDK is lazily installed once (prompt=False, gated by + security.allow_lazy_installs); if it still can't load, log and no-op — never raise into startup. """ if not is_enabled(config): return None @@ -329,11 +288,5 @@ def start_streaming( __all__ = [ - "OTLPUnavailable", - "OTLPStreamer", - "build_exporter", - "export_batch", - "is_available", - "is_enabled", - "start_streaming", + "OTLPUnavailable", "OTLPStreamer", "build_exporter", "export_batch", "is_available", "is_enabled", "start_streaming", ] diff --git a/agent/monitoring/policy.py b/agent/monitoring/policy.py index 0939fbbc67..a50f397c7a 100644 --- a/agent/monitoring/policy.py +++ b/agent/monitoring/policy.py @@ -46,6 +46,4 @@ def ensure_install_id(config: Dict[str, Any]) -> str: return minted -__all__ = [ - "ensure_install_id", -] +__all__ = ["ensure_install_id"] diff --git a/agent/monitoring/redaction.py b/agent/monitoring/redaction.py index 22ae1c0904..f716c00f08 100644 --- a/agent/monitoring/redaction.py +++ b/agent/monitoring/redaction.py @@ -1,11 +1,9 @@ """Redaction applied to monitoring data before egress. -One unconditional scrub, no modes, no knobs. Every string that leaves the -process passes through ``redact_for_export``: secrets first (wraps -``agent/redact.py::redact_sensitive_text(force=True)`` plus bearer/token -shapes, failing CLOSED so a broken redactor never emits the raw string), then +One unconditional scrub, no modes, no knobs. Every string that leaves the process passes +through ``redact_for_export``: secrets first (``agent/redact.py::redact_sensitive_text(force=True)`` +plus bearer/token shapes, failing CLOSED so a broken redactor never emits the raw string), then PII (e-mail, phone, UUID-shaped ids -> ``[email]`` / ``[phone]`` / ``[id]``). -There is deliberately no setting to weaken this. """ from __future__ import annotations @@ -15,9 +13,7 @@ from typing import Any, Optional # ── secret shapes (belt-and-suspenders on top of agent/redact.py) ─────────── _BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE) -_TOKEN_RE = re.compile( - r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b" -) +_TOKEN_RE = re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b") _SECRET_LITERAL_RE = re.compile(r"\*{3,}") _BEARER_RESIDUE_RE = re.compile(r"\bBearer\s+\[[^\]]+\]", re.IGNORECASE) @@ -27,9 +23,7 @@ _EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") _PHONE_RE = re.compile( r"(? Optional[str]: return out -def redact_bounded( - raw: Any, - *, - limit: int = 500, - empty: str = "[redacted]", - unavailable: str = UNAVAILABLE, -) -> str: +def redact_bounded(raw: Any, *, limit: int = 500, empty: str = "[redacted]", unavailable: str = UNAVAILABLE) -> str: """Redact ``str(raw or "")`` and length-bound it; ``empty`` replaces an empty result, ``unavailable`` is returned if redaction itself raises.""" try: @@ -73,7 +61,4 @@ def redact_bounded( return unavailable -__all__ = [ - "redact_for_export", - "redact_bounded", -] +__all__ = ["redact_for_export", "redact_bounded"] From ebd1814862f3daa6c166ef621a3b85014c6b5b9e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:34:52 -0700 Subject: [PATCH 4/9] =?UTF-8?q?refactor(agent):=20iron=5Fproxy=20second=20?= =?UTF-8?q?pass=20=E2=80=94=20single-line=20diagnostics,=20alias-source=20?= =?UTF-8?q?lookup,=20suppress()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/proxy_sources/iron_proxy.py | 120 +++++++++--------------------- 1 file changed, 35 insertions(+), 85 deletions(-) diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index 5a055c2e82..6ab6c18aba 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -207,8 +207,7 @@ def _platform_asset_name() -> str: return f"iron-proxy_{_IRON_PROXY_VERSION}_{os_name}_{arch}.tar.gz" if system == "Windows": raise RuntimeError( - f"iron-proxy does not ship native Windows binaries as of v{_IRON_PROXY_VERSION}. " - "Run the proxy on a Linux/macOS host, or inside WSL." + f"iron-proxy does not ship native Windows binaries as of v{_IRON_PROXY_VERSION}. Run the proxy on a Linux/macOS host, or inside WSL." ) raise RuntimeError(f"Unsupported platform for iron-proxy auto-install: {system} {machine}") @@ -293,10 +292,7 @@ def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: is never a hard dependency. Raises ONLY on a present-but-bad signature (tamper signal).""" gpg = shutil.which("gpg") if not gpg: - logger.warning( - "gpg not found on PATH — skipping iron-proxy release-signature " - "verification (SHA-256 checksum check still enforced)." - ) + logger.warning("gpg not found on PATH — skipping iron-proxy release-signature verification (SHA-256 checksum check still enforced).") return False sig_path, pubkey_path = tmp / _IRON_PROXY_CHECKSUM_SIG_NAME, tmp / _IRON_PROXY_PUBKEY_NAME @@ -304,10 +300,7 @@ def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: _release_asset(_IRON_PROXY_CHECKSUM_SIG_NAME, sig_path) _release_asset(_IRON_PROXY_PUBKEY_NAME, pubkey_path) except RuntimeError as exc: - logger.warning( - "iron-proxy release signature assets unavailable (%s) — skipping " - "GPG verification (SHA-256 checksum check still enforced).", exc, - ) + logger.warning("iron-proxy release signature assets unavailable (%s) — skipping GPG verification (SHA-256 checksum check still enforced).", exc) return False gnupg_home = tmp / "gnupg" @@ -315,17 +308,13 @@ def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: gpg_base = [gpg, "--homedir", str(gnupg_home), "--batch", "--no-tty"] imp = _run([*gpg_base, "--import", str(pubkey_path)], timeout=60) if imp.returncode != 0: - logger.warning( - "Could not import iron-proxy signing key — skipping GPG verification (SHA-256 still enforced): %s", - imp.stderr.decode("utf-8", "replace")[:200], - ) + logger.warning("Could not import iron-proxy signing key — skipping GPG verification (SHA-256 still enforced): %s", imp.stderr.decode("utf-8", "replace")[:200]) return False verify = _run([*gpg_base, "--verify", str(sig_path), str(checksum_path)], timeout=60) if verify.returncode != 0: raise RuntimeError( - "iron-proxy checksums.txt failed GPG signature verification — " - "refusing to install (possible release-channel tampering). " - f"gpg: {verify.stderr.decode('utf-8', 'replace')[:300]}" + "iron-proxy checksums.txt failed GPG signature verification — refusing to install (possible release-channel " + f"tampering). gpg: {verify.stderr.decode('utf-8', 'replace')[:300]}" ) logger.info("Verified iron-proxy checksums.txt GPG signature.") return True @@ -356,10 +345,7 @@ def _pick_tar_member(tf: tarfile.TarFile, binary_name: str) -> tarfile.TarInfo: and Path(m.name).name == binary_name ] if not candidates: - raise RuntimeError( - f"Could not find {binary_name} inside downloaded archive " - f"(members: {[m.name for m in tf.getmembers()[:5]]}...)" - ) + raise RuntimeError(f"Could not find {binary_name} inside downloaded archive (members: {[m.name for m in tf.getmembers()[:5]]}...)") return min(candidates, key=lambda m: len(m.name)) @@ -419,9 +405,7 @@ def ensure_ca_cert(*, force: bool = False) -> Tuple[Path, Path]: if ca_crt.exists() and ca_key.exists() and not force: return ca_crt, ca_key if shutil.which("openssl") is None: - raise RuntimeError( - "openssl not found on PATH. Install OpenSSL (apt: `openssl`, brew: `openssl`) to generate the iron-proxy CA cert." - ) + raise RuntimeError("openssl not found on PATH. Install OpenSSL (apt: `openssl`, brew: `openssl`) to generate the iron-proxy CA cert.") with tempfile.TemporaryDirectory(prefix="hermes-proxy-ca-") as tmpdir: tmp_key, tmp_crt = Path(tmpdir) / "ca.key", Path(tmpdir) / "ca.crt" @@ -546,10 +530,9 @@ def reload_proxy() -> bool: return True raise RuntimeError(f"management API returned unexpected status {resp.status}") except urllib.error.HTTPError as exc: - try: + body = "" + with suppress(OSError): body = exc.read().decode("utf-8", errors="replace")[:500] - except OSError: - body = "" message = _RELOAD_HTTP_ERRORS.get(exc.code, "management reload failed (HTTP {code}): {body}") raise RuntimeError(message.format(code=exc.code, body=body)) from exc except (urllib.error.URLError, OSError) as exc: @@ -690,8 +673,7 @@ def ensure_audit_log(audit_path: Path) -> None: os.close(_open_private_append(audit_path, strict_chmod=True)) except OSError as exc: raise RuntimeError( - f"Refusing to start: could not pre-create audit log " - f"{audit_path} with restrictive permissions ({exc}). " + f"Refusing to start: could not pre-create audit log {audit_path} with restrictive permissions ({exc}). " f"Move or chmod any existing file at that path and retry." ) from exc @@ -746,14 +728,10 @@ def load_mappings() -> List[TokenMapping]: return [] out: List[TokenMapping] = [] for item in payload.get("tokens", []): - try: + try: # pre-header-auth files load with the bearer defaults they were written under out.append(TokenMapping( - proxy_token=item["proxy_token"], - real_env_name=item["env_name"], - upstream_hosts=tuple(item.get("upstream_hosts") or ()), - # Pre-header-auth files load with the bearer defaults they were written under. - match_headers=tuple(item.get("match_headers") or ("Authorization",)), - alias_env_names=tuple(item.get("alias_env_names") or ()), + item["proxy_token"], item["env_name"], tuple(item.get("upstream_hosts") or ()), + tuple(item.get("match_headers") or ("Authorization",)), tuple(item.get("alias_env_names") or ()), )) except (KeyError, TypeError): continue @@ -762,9 +740,7 @@ def load_mappings() -> List[TokenMapping]: def _env_names(available_env_names: Optional[List[str]]) -> set: """Explicit override (Bitwarden adapter) or the non-empty names in the host env.""" - if available_env_names is not None: - return set(available_env_names) - return {k for k, v in os.environ.items() if v} + return set(available_env_names) if available_env_names is not None else {k for k, v in os.environ.items() if v} def discover_provider_mappings(*, available_env_names: Optional[List[str]] = None) -> List[TokenMapping]: @@ -936,8 +912,7 @@ def start_proxy( return _abort(f"iron-proxy exited immediately (code {proc.returncode}). ", kill=False) def _interrupt_handler(_signum, _frame): # pragma: no cover - signal path - # Ctrl-C while waiting must not leak an orphan holding the port. - _kill_and_wait(proc, grace_seconds=2) + _kill_and_wait(proc, grace_seconds=2) # Ctrl-C while waiting must not leak an orphan holding the port pidfile.unlink(missing_ok=True) raise KeyboardInterrupt() @@ -961,10 +936,7 @@ def start_proxy( raise _exited_error() # Alive-but-not-listening is a failure: an orphan holding the port breaks every restart. if not listening: - raise _abort( - f"iron-proxy did not bind {probe_host}:{tunnel_port} within {_STARTUP_GRACE_SECONDS}s. Process was killed. ", - kill=True, - ) + raise _abort(f"iron-proxy did not bind {probe_host}:{tunnel_port} within {_STARTUP_GRACE_SECONDS}s. Process was killed. ", kill=True) logger.info("Started iron-proxy pid=%s config=%s", proc.pid, cfg) return get_status() @@ -1020,10 +992,7 @@ def _write_pidfile_safely(pidfile: Path, pid: int) -> None: except FileExistsError: existing_pid = _read_pid() if existing_pid and _pid_alive(existing_pid): - raise RuntimeError( - f"Another iron-proxy start appears to be in progress (pidfile {pidfile} -> pid {existing_pid}). " - f"Run `hermes egress stop` if that proxy is stuck." - ) + raise RuntimeError(f"Another iron-proxy start appears to be in progress (pidfile {pidfile} -> pid {existing_pid}). Run `hermes egress stop` if that proxy is stuck.") pidfile.unlink(missing_ok=True) fd = os.open(str(pidfile), open_flags, 0o600) except OSError as exc: @@ -1074,13 +1043,9 @@ def _build_proxy_subprocess_env( needed = {m.real_env_name for m in mappings} alias_sources = {m.real_env_name: m.alias_env_names for m in mappings if m.alias_env_names} for name in needed: - if name in parent: - env[name] = parent[name] - else: - for alias in alias_sources.get(name, ()): - if parent.get(alias): - env[name] = parent[alias] - break + source = name if name in parent else next((a for a in alias_sources.get(name, ()) if parent.get(a)), None) + if source is not None: + env[name] = parent[source] if refresh_from_bitwarden and bitwarden_config: _refresh_secrets_from_bitwarden(env, needed, bitwarden_config, bool(bitwarden_config.get("allow_env_fallback"))) @@ -1114,16 +1079,12 @@ def _refresh_secrets_from_bitwarden(env: Dict[str, str], needed: set, bitwarden_ # Don't interpolate access_token_name — CodeQL treats config values as tainted. _bitwarden_shortfall( allow_env_fallback, - "credential_source=bitwarden but the access-token env or project_id is empty. Either set both, " - "switch to credential_source: env, or set `proxy.allow_env_fallback: true` to opt into " - "the legacy fallback behaviour.", - "credential_source=bitwarden but access-token env or project_id is empty — " - "proxy will fall back to parent env (allow_env_fallback=true).", + "credential_source=bitwarden but the access-token env or project_id is empty. Either set both, switch to " + "credential_source: env, or set `proxy.allow_env_fallback: true` to opt into the legacy fallback behaviour.", + "credential_source=bitwarden but access-token env or project_id is empty — proxy will fall back to parent env (allow_env_fallback=true).", ) return - secrets, warnings = bw.fetch_bitwarden_secrets( - access_token=access_token, project_id=project_id, cache_ttl_seconds=0, use_cache=False, - ) + secrets, warnings = bw.fetch_bitwarden_secrets(access_token=access_token, project_id=project_id, cache_ttl_seconds=0, use_cache=False) except ImportError as exc: # A dependency vanishing between setup and restart must not silently degrade. if not allow_env_fallback: @@ -1132,10 +1093,7 @@ def _refresh_secrets_from_bitwarden(env: Dict[str, str], needed: set, bitwarden_ "proxy.allow_env_fallback: false). Either fix the import, switch to credential_source: env, or set " "`proxy.allow_env_fallback: true` to opt into the legacy fallback behaviour." ) from exc - logger.warning( - "Bitwarden refresh module unavailable at proxy start, falling back to parent env (allow_env_fallback=true): %s", - exc, - ) + logger.warning("Bitwarden refresh module unavailable at proxy start, falling back to parent env (allow_env_fallback=true): %s", exc) return missing = sorted(needed - set(secrets)) @@ -1145,18 +1103,14 @@ def _refresh_secrets_from_bitwarden(env: Dict[str, str], needed: set, bitwarden_ if missing: _bitwarden_shortfall( allow_env_fallback, - f"Bitwarden refresh did not return secrets for {missing}. Either add the secrets to your BWS " - f"project, switch to credential_source: env via `hermes egress setup --no-bitwarden`, or set " - f"`proxy.allow_env_fallback: true` in config.yaml to opt into the legacy host-env fallback.", - "Bitwarden refresh did not return secrets for %s — falling back to host env for those names " - "(allow_env_fallback=true).", + f"Bitwarden refresh did not return secrets for {missing}. Either add the secrets to your BWS project, switch to " + f"credential_source: env via `hermes egress setup --no-bitwarden`, or set `proxy.allow_env_fallback: true` in " + f"config.yaml to opt into the legacy host-env fallback.", + "Bitwarden refresh did not return secrets for %s — falling back to host env for those names (allow_env_fallback=true).", missing, ) - # Log only the count: the taint analyzer can't tell bws status text is non-secret. - if warnings: - logger.warning( - "Bitwarden refresh produced %d warning(s); run `hermes secrets bitwarden status` for detail.", len(warnings), - ) + if warnings: # log only the count: the taint analyzer can't tell bws status text is non-secret + logger.warning("Bitwarden refresh produced %d warning(s); run `hermes secrets bitwarden status` for detail.", len(warnings)) def _forget_daemon() -> None: @@ -1191,9 +1145,7 @@ def stop_proxy() -> bool: time.sleep(0.1) else: starttime_after = _pid_proc_starttime(pid) - recycled = ( - starttime_before is not None and starttime_after is not None and starttime_before != starttime_after - ) or not _pid_alive(pid) + recycled = (starttime_before is not None and starttime_after is not None and starttime_before != starttime_after) or not _pid_alive(pid) if recycled: logger.warning("iron-proxy pid=%s appears recycled before SIGKILL; not killing.", pid) else: @@ -1219,8 +1171,7 @@ def get_status() -> ProxyStatus: status.ca_cert_path = ca if ca.exists() else None pid = _read_pid() if pid and _pid_alive(pid): - status.pid = pid - status.listening = _port_listening(probe_host, status.tunnel_port) + status.pid, status.listening = pid, _port_listening(probe_host, status.tunnel_port) return status @@ -1238,8 +1189,7 @@ def _tail_log(path: Path, *, lines: int = 20) -> str: if not path.exists(): return "(no log file)" try: - data = path.read_bytes()[-8192:] - return "\n".join(data.decode("utf-8", errors="replace").splitlines()[-lines:]) + return "\n".join(path.read_bytes()[-8192:].decode("utf-8", errors="replace").splitlines()[-lines:]) except OSError as exc: return f"(could not read log: {exc})" From 79c704f34f65ec1e71fcc9fa31fe2484f0989b5e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:48:19 -0700 Subject: [PATCH 5/9] =?UTF-8?q?refactor(agent):=20monitoring/iron=5Fproxy?= =?UTF-8?q?=20third=20pass=20=E2=80=94=20gateway.status=20dispatch=20helpe?= =?UTF-8?q?r,=20SDK=20symbol=20table=20by=20module,=20compact=20comments?= =?UTF-8?q?=20and=20inner=20blanks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/monitoring/cron_health.py | 2 +- agent/monitoring/events.py | 6 - agent/monitoring/gateway_health.py | 97 ++++------ agent/monitoring/gateway_health_export.py | 27 +-- agent/monitoring/otlp_exporter.py | 51 +++-- agent/monitoring/policy.py | 2 - agent/outbound_webhooks.py | 12 -- agent/proxy_sources/iron_proxy.py | 223 ++++++++-------------- 8 files changed, 150 insertions(+), 270 deletions(-) diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py index 40a2344455..de4e9ea73d 100644 --- a/agent/monitoring/cron_health.py +++ b/agent/monitoring/cron_health.py @@ -100,7 +100,6 @@ def emit_execution_state(record: Optional[dict[str, Any]], *, delivery_outcome: return try: from agent.monitoring import emitter - event = project_execution_event(record, delivery_outcome=delivery_outcome) target = emitter.get_emitter() target.emit(event) @@ -136,6 +135,7 @@ def _freshness_metric(name: str, reader: Callable[[], Optional[float]]) -> Calla value = reader() if value is not None: metrics.append(GatewayMetric(name, max(0.0, float(value)), {})) + return build diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py index a9bb57dcb5..877a537fd3 100644 --- a/agent/monitoring/events.py +++ b/agent/monitoring/events.py @@ -24,9 +24,7 @@ class _MonitoringEvent: @dataclass(slots=True) class GatewayHealthEvent(_MonitoringEvent): """Content-free gateway health snapshot or lifecycle event.""" - EVENT: ClassVar[str] = "gateway_health" - name: str gateway_state: Optional[str] = None old_state: Optional[str] = None @@ -49,9 +47,7 @@ class GatewayHealthEvent(_MonitoringEvent): @dataclass(slots=True) class GatewayDiagnosticEvent(_MonitoringEvent): """Redacted gateway diagnostic event for operator-owned observability.""" - EVENT: ClassVar[str] = "gateway_diagnostic" - name: str subsystem: str error_class: str = "unknown" @@ -69,9 +65,7 @@ class GatewayDiagnosticEvent(_MonitoringEvent): @dataclass(slots=True) class CronExecutionEvent(_MonitoringEvent): """Content-free durable cron execution lifecycle projection.""" - EVENT: ClassVar[str] = "cron_execution" - status: str job_key: str source: str = "unknown" diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index ea7ca161ac..c305110896 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -126,31 +126,42 @@ def platform_for_subsystem(subsystem: str) -> Optional[str]: return (subsystem.split(".", 1)[1] or None) if subsystem.startswith("platform.") else None -def _parse_active_agents(raw: Any) -> int: +def _gateway_status(name: str, fallback: Callable[..., Any], /, **kwargs: Any) -> Any: + """Prefer ``gateway.status.`` (the runtime-status contract); fall back to the local approximation.""" try: - from gateway.status import parse_active_agents - return parse_active_agents(raw) + import gateway.status as status + return getattr(status, name)(**kwargs) except Exception: - try: - return max(0, int(raw)) - except (TypeError, ValueError): - return 0 + return fallback(**kwargs) + + +def _int_or_zero(raw: Any) -> int: + try: + return max(0, int(raw)) + except (TypeError, ValueError): + return 0 + + +def _parse_active_agents(raw: Any) -> int: + return _gateway_status("parse_active_agents", _int_or_zero, raw=raw) def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: - try: - from gateway.status import derive_gateway_busy - return derive_gateway_busy(gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents) - except Exception: - return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0) + return _gateway_status( + "derive_gateway_busy", + lambda gateway_running, gateway_state, active_agents: bool( + gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0 + ), + gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents, + ) def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool: - try: - from gateway.status import derive_gateway_drainable - return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state) - except Exception: - return bool(gateway_running and gateway_state == "running") + return _gateway_status( + "derive_gateway_drainable", + lambda gateway_running, gateway_state: bool(gateway_running and gateway_state == "running"), + gateway_running=gateway_running, gateway_state=gateway_state, + ) def _base_attrs(*, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]: @@ -181,12 +192,7 @@ def _platform_error_code(pdata: dict[str, Any]) -> str: def build_gateway_health_snapshot( - runtime: Optional[dict[str, Any]], - *, - gateway_running: bool, - profile: str, - install_id: str, - version: str, + runtime: Optional[dict[str, Any]], *, gateway_running: bool, profile: str, install_id: str, version: str, supervision_mode: str = "unknown", ) -> GatewayHealthSnapshot: """Convert gateway_state.json-compatible runtime state into P0 signals.""" @@ -197,7 +203,6 @@ def build_gateway_health_snapshot( drainable = _derive_drainable(gateway_running, gateway_state) platforms = _platforms_of(runtime) base = _base_attrs(install_id=install_id, version=version, supervision_mode=supervision_mode) - metrics: list[GatewayMetric] = [ _metric("hermes.gateway.up", 1 if gateway_running else 0, base), _metric("hermes.gateway.active_agents", active_agents, base), @@ -206,7 +211,6 @@ def build_gateway_health_snapshot( _metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base), _metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": gateway_state}), ] - fatal_count = 0 events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] for platform, pdata in platforms.items(): @@ -227,20 +231,10 @@ def build_gateway_health_snapshot( error_code=error_code, error_class=error_code, profile=profile, version=version, severity="error" if state == "fatal" else "warning", )) - events.insert(0, GatewayHealthEvent( - name="gateway.health_snapshot", - gateway_state=gateway_state, - active_agents=active_agents, - gateway_busy=busy, - gateway_drainable=drainable, - platform_count=len(platforms), - fatal_platform_count=fatal_count, - profile=profile, - install_id=install_id, - version=version, - supervision_mode=supervision_mode, - pid=_coerce_pid(runtime.get("pid")), + name="gateway.health_snapshot", gateway_state=gateway_state, active_agents=active_agents, gateway_busy=busy, + gateway_drainable=drainable, platform_count=len(platforms), fatal_platform_count=fatal_count, profile=profile, + install_id=install_id, version=version, supervision_mode=supervision_mode, pid=_coerce_pid(runtime.get("pid")), )) return GatewayHealthSnapshot(metrics=metrics, events=events) @@ -273,16 +267,10 @@ def _lifecycle_events( def health(name: str) -> GatewayHealthEvent: return GatewayHealthEvent( - name=name, - gateway_state=new_state, - old_state=old_state, - new_state=new_state, + name=name, gateway_state=new_state, old_state=old_state, new_state=new_state, exit_reason=classify_exit_reason(current.get("exit_reason"), state=new_state, restart_requested=restart_requested), - restart_requested=restart_requested, - active_agents=_parse_active_agents(current.get("active_agents", 0)), - profile=profile, - version=version, - pid=_coerce_pid(current.get("pid")), + restart_requested=restart_requested, active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, version=version, pid=_coerce_pid(current.get("pid")), ) out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [health("gateway.lifecycle")] @@ -351,21 +339,14 @@ class GatewayDiagnosticLogHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: try: - if record.levelno < logging.WARNING: - return - if not (record.name == "gateway" or record.name.startswith("gateway.")): + if record.levelno < logging.WARNING or not (record.name == "gateway" or record.name.startswith("gateway.")): return subsystem = subsystem_for_logger(record.name) error_class = classify_gateway_error(record.getMessage()) emitter.get_emitter().emit(GatewayDiagnosticEvent( - name=f"gateway.log.{record.levelname.lower()}", - subsystem=subsystem, - source_logger=source_logger_for_export(record.name), - platform=platform_for_subsystem(subsystem), - error_class=error_class, - error_code=error_class, - profile=self.profile, - version=self.version, + name=f"gateway.log.{record.levelname.lower()}", subsystem=subsystem, + source_logger=source_logger_for_export(record.name), platform=platform_for_subsystem(subsystem), + error_class=error_class, error_code=error_class, profile=self.profile, version=self.version, severity=record.levelname.lower(), )) except Exception: diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 81f4bfd95d..643298be61 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -80,7 +80,6 @@ class GatewayHealthExportRuntime: if self.log_handler is not None: with suppress(Exception): logging.getLogger().removeHandler(self.log_handler) - # Producers are stopped; drain queued/in-flight events BEFORE detaching subscribers # so the terminal lifecycle event cannot race exporter shutdown. Bounded, fail-open. subscribers = [item for item in (self.streamer, self.log_streamer) if item is not None] @@ -89,7 +88,6 @@ class GatewayHealthExportRuntime: bus.flush(timeout=1.0) for sub in subscribers: bus.unsubscribe(sub) - # Network flush/close runs under one bounded daemon-thread deadline so it can # never delay gateway teardown indefinitely. closeables = subscribers + ([self.metric_provider] if self.metric_provider is not None else []) @@ -103,7 +101,6 @@ class GatewayHealthExportRuntime: worker = threading.Thread(target=_close, name="hermes-gateway-health-export-shutdown", daemon=True) worker.start() worker.join(timeout=2.0) - self.streamer = self.log_streamer = self.metric_provider = self.thread = self.stop_event = None @@ -161,7 +158,6 @@ def _read_gateway_snapshot(config: Dict[str, Any]): def _read_cron_snapshot(): from agent.monitoring.cron_health import build_cron_health_snapshot - return build_cron_health_snapshot() @@ -242,11 +238,11 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: def callback(name: str): def _cb(_options=None): try: - snapshot = _read_runtime_snapshot(config) - return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name] + return [Observation(m.value, m.attributes) for m in _read_runtime_snapshot(config).metrics if m.name == name] except Exception: logger.debug("gateway metric callback failed", exc_info=True) return [] + return _cb for metric_name in _OBSERVABLE_METRIC_NAMES: @@ -286,13 +282,9 @@ class GatewayDiagnosticLogStreamer(EmitterStreamer): source_logger = source_logger_for_export(ev.get("source_logger")) otel_logger = self._provider.get_logger(source_logger) if source_logger is not None else self._logger otel_logger.emit(sdk["LogRecord"]( - timestamp=ev.get("ts_ns"), - trace_id=sdk["INVALID_TRACE_ID"], - span_id=sdk["INVALID_SPAN_ID"], - trace_flags=sdk["TraceFlags"].DEFAULT, - severity_text=str(ev.get("severity") or "warning").upper(), - severity_number=_severity_number(sdk, ev.get("severity")), - body=redact_bounded("gateway diagnostic"), + timestamp=ev.get("ts_ns"), trace_id=sdk["INVALID_TRACE_ID"], span_id=sdk["INVALID_SPAN_ID"], + trace_flags=sdk["TraceFlags"].DEFAULT, severity_text=str(ev.get("severity") or "warning").upper(), + severity_number=_severity_number(sdk, ev.get("severity")), body=redact_bounded("gateway diagnostic"), attributes=_diagnostic_log_attributes(ev), )) self.exported += 1 @@ -334,17 +326,12 @@ def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRu diagnostics_on = gh.get("diagnostic_events_enabled", True) runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled") sdk: Optional[Dict[str, Any]] = None - if metrics_on or diagnostics_on: try: sdk = _require_metrics_sdk(prompt=False) except Exception: - logger.warning( - "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; install 'hermes-agent[otlp]'", - exc_info=True, - ) + logger.warning("monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; install 'hermes-agent[otlp]'", exc_info=True) return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable") - if metrics_on and sdk is not None: try: runtime.metric_provider = _start_metric_provider(config, sdk) @@ -352,7 +339,6 @@ def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRu logger.warning("gateway health OTLP metrics failed to start", exc_info=True) runtime.shutdown() return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed") - if diagnostics_on and sdk is not None: try: runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event) @@ -365,7 +351,6 @@ def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRu logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True) runtime.shutdown() return GatewayHealthExportRuntime(enabled=False, reason="diagnostics_start_failed") - try: runtime.log_handler = _attach_log_handler(config) except Exception: diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index 0c2d8e102c..2f845569ff 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -29,25 +29,23 @@ class OTLPUnavailable(RuntimeError): # ── SDK loading ────────────────────────────────────────────────────────────── -_SDK_SYMBOLS: Dict[str, str] = { - "TracerProvider": "opentelemetry.sdk.trace", - "BatchSpanProcessor": "opentelemetry.sdk.trace.export", - "Resource": "opentelemetry.sdk.resources", - "OTLPSpanExporter": "opentelemetry.exporter.otlp.proto.http.trace_exporter", - "SpanKind": "opentelemetry.trace", - "OTLPLogExporter": "opentelemetry.exporter.otlp.proto.http._log_exporter", - "OTLPMetricExporter": "opentelemetry.exporter.otlp.proto.http.metric_exporter", - "Observation": "opentelemetry.metrics", - "INVALID_SPAN_ID": "opentelemetry.trace", - "INVALID_TRACE_ID": "opentelemetry.trace", - "TraceFlags": "opentelemetry.trace", - "LogRecord": "opentelemetry._logs", - "SeverityNumber": "opentelemetry._logs.severity", - "LoggerProvider": "opentelemetry.sdk._logs", - "BatchLogRecordProcessor": "opentelemetry.sdk._logs.export", - "MeterProvider": "opentelemetry.sdk.metrics", - "PeriodicExportingMetricReader": "opentelemetry.sdk.metrics.export", +_SDK_MODULES: Dict[str, tuple[str, ...]] = { + "opentelemetry.sdk.trace": ("TracerProvider",), + "opentelemetry.sdk.trace.export": ("BatchSpanProcessor",), + "opentelemetry.sdk.resources": ("Resource",), + "opentelemetry.exporter.otlp.proto.http.trace_exporter": ("OTLPSpanExporter",), + "opentelemetry.exporter.otlp.proto.http._log_exporter": ("OTLPLogExporter",), + "opentelemetry.exporter.otlp.proto.http.metric_exporter": ("OTLPMetricExporter",), + "opentelemetry.trace": ("SpanKind", "INVALID_SPAN_ID", "INVALID_TRACE_ID", "TraceFlags"), + "opentelemetry.metrics": ("Observation",), + "opentelemetry._logs": ("LogRecord",), + "opentelemetry._logs.severity": ("SeverityNumber",), + "opentelemetry.sdk._logs": ("LoggerProvider",), + "opentelemetry.sdk._logs.export": ("BatchLogRecordProcessor",), + "opentelemetry.sdk.metrics": ("MeterProvider",), + "opentelemetry.sdk.metrics.export": ("PeriodicExportingMetricReader",), } +_SDK_SYMBOLS: Dict[str, str] = {name: module for module, names in _SDK_MODULES.items() for name in names} _SPAN_SDK = ("TracerProvider", "BatchSpanProcessor", "Resource", "OTLPSpanExporter", "SpanKind") @@ -172,16 +170,12 @@ def _make_provider(config: Dict[str, Any]): # ── event -> span attribute mapping ────────────────────────────────────────── # Per-kind attribute allowlists: everything else (profile, install_id, ...) never egresses. _KEEP_BY_KIND: Dict[str, tuple[str, ...]] = { - "gateway_health": ("name", "gateway_state", "old_state", "new_state", - "exit_reason", "restart_requested", "active_agents", - "gateway_busy", "gateway_drainable", "platform_count", - "fatal_platform_count", "version", - "supervision_mode", "pid"), - "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", - "platform", "old_state", "new_state", - "version", "severity"), - "cron_execution": ("status", "job_key", "source", "duration_ms", - "delivery_outcome", "error_class"), + "gateway_health": ( + "name", "gateway_state", "old_state", "new_state", "exit_reason", "restart_requested", "active_agents", + "gateway_busy", "gateway_drainable", "platform_count", "fatal_platform_count", "version", "supervision_mode", "pid", + ), + "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", "platform", "old_state", "new_state", "version", "severity"), + "cron_execution": ("status", "job_key", "source", "duration_ms", "delivery_outcome", "error_class"), } @@ -220,7 +214,6 @@ def export_batch(provider, batch: List[Dict[str, Any]]) -> int: class EmitterStreamer: """Base for emitter subscribers owning an OTel provider + batch processor. Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.""" - _provider: Any _processor: Any exported: int = 0 diff --git a/agent/monitoring/policy.py b/agent/monitoring/policy.py index a50f397c7a..1c1620a04f 100644 --- a/agent/monitoring/policy.py +++ b/agent/monitoring/policy.py @@ -25,11 +25,9 @@ def ensure_install_id(config: Dict[str, Any]) -> str: existing = (mon or {}).get("install_id") if isinstance(mon, dict) else None if isinstance(existing, str) and existing.strip(): return existing - minted = str(uuid.uuid4()) try: from hermes_cli.config import load_config, save_config - fresh = load_config() if isinstance(fresh, dict): slot = fresh.setdefault("monitoring", {}) diff --git a/agent/outbound_webhooks.py b/agent/outbound_webhooks.py index 4604504147..4504838888 100644 --- a/agent/outbound_webhooks.py +++ b/agent/outbound_webhooks.py @@ -59,9 +59,7 @@ _worker: Optional[threading.Thread] = None @dataclass class WebhookTarget(_ToolMatcherMixin): """Parsed and validated representation of one ``hooks.outbound`` entry.""" - _MATCHER_KIND = "outbound webhook" - url: str events: List[str] name: str = "" @@ -86,7 +84,6 @@ def register_from_config(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: if not isinstance(cfg, dict): return [] from utils import env_var_enabled - if env_var_enabled("HERMES_SAFE_MODE"): logger.info("HERMES_SAFE_MODE=1 — outbound webhook registration skipped") return [] @@ -94,7 +91,6 @@ def register_from_config(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: if not targets: return [] from hermes_cli.plugins import get_plugin_manager - manager = get_plugin_manager() home_key = _home_key() registered: List[WebhookTarget] = [] @@ -149,7 +145,6 @@ def re_register_config_hooks() -> None: current home's idempotence keys are cleared so a force-reload in one profile cannot invalidate another profile's still-live registration.""" from hermes_cli.config import load_config - _forget_home_registrations(_registered, _registered_lock) register_from_config(load_config()) @@ -177,7 +172,6 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: if not isinstance(raw, dict): warn(" must be a mapping with 'url' and 'events' keys; got %s", type(raw).__name__) return None - url = raw.get("url") if not isinstance(url, str) or not url.strip(): warn(" is missing a non-empty 'url'") @@ -188,7 +182,6 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: return None if url.lower().startswith("http://"): warn(".url uses plain http:// — payloads (including tool inputs) travel unencrypted. Prefer https.") - events_raw = raw.get("events") valid_list = ", ".join(sorted(VALID_HOOKS)) if not isinstance(events_raw, list) or not events_raw: @@ -203,7 +196,6 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: if not events: warn(" has no valid events — skipped") return None - matcher = raw.get("matcher") if matcher is not None and not isinstance(matcher, str): warn(".matcher must be a string regex; ignoring") @@ -211,14 +203,12 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: if matcher is not None and not any(e in _TOOL_SCOPED_EVENTS for e in events): warn(".matcher=%r will be ignored — matcher is only honored for pre_tool_call / post_tool_call.", matcher) matcher = None - timeout_raw = raw.get("timeout", DEFAULT_TIMEOUT_SECONDS) try: timeout = int(timeout_raw) except (TypeError, ValueError): warn(".timeout must be an int (got %r); using default %ds", timeout_raw, DEFAULT_TIMEOUT_SECONDS) timeout = DEFAULT_TIMEOUT_SECONDS - name = raw.get("name") return WebhookTarget( url=url, @@ -276,7 +266,6 @@ def _serialize_payload(event: str, kwargs: Dict[str, Any], delivery_id: str) -> body, so they double as replay protection.""" # Profile resolved at fire time so a multiplexed gateway's receivers can tell which profile emitted. from hermes_cli.profiles import get_active_profile_name - payload = { "hook_event_name": event, "profile": get_active_profile_name(), @@ -383,7 +372,6 @@ def _deliver(delivery: Dict[str, Any]) -> None: last_error = str(exc) or type(exc).__name__ if attempt < MAX_DELIVERY_ATTEMPTS: time.sleep(RETRY_BACKOFF_SECONDS * attempt) - logger.warning( "outbound webhook delivery failed after %d attempt(s) (event=%s target=%s): %s", MAX_DELIVERY_ATTEMPTS, event, label, last_error, diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index 6ab6c18aba..d5cd271165 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -31,13 +31,11 @@ from typing import Dict, List, Optional, Tuple logger = logging.getLogger(__name__) -# Pinned upstream version. Never auto-resolve "latest": the YAML schema may change between -# releases and updates must be deliberate. +# Pinned: never auto-resolve "latest" — the YAML schema may change between releases. _IRON_PROXY_VERSION = "0.39.0" _IRON_PROXY_RELEASE_BASE = f"https://github.com/ironsh/iron-proxy/releases/download/v{_IRON_PROXY_VERSION}" _IRON_PROXY_CHECKSUM_NAME = "checksums.txt" -# Detached signature + signing key for optional GPG verification of checksums.txt (SHA-256 -# alone only protects the archive if checksums.txt itself came from an untampered channel). +# Optional GPG verification of checksums.txt (SHA-256 alone trusts the release channel). _IRON_PROXY_CHECKSUM_SIG_NAME = "checksums.txt.asc" _IRON_PROXY_PUBKEY_NAME = "public-key.asc" @@ -45,9 +43,8 @@ _DOWNLOAD_TIMEOUT = 120 # binary is ~16MB _RUN_TIMEOUT = 30 _STARTUP_GRACE_SECONDS = 5 -# Management API (v0.39): authenticated loopback listener whose POST /v1/reload hot-swaps the -# ruleset. Bearer key minted at setup, stored 0600 at /management.token, injected into -# the daemon env under this name; v0.39 refuses to start if the named var is empty. +# Management API (v0.39): loopback POST /v1/reload hot-swaps the ruleset. Bearer key minted at +# setup (0600 at /management.token), injected under this env name; empty => daemon refuses to start. _MGMT_API_KEY_ENV = "HERMES_IRON_PROXY_MGMT_KEY" _MGMT_PORT_OFFSET = 2 # tunnel_port is CONNECT/MITM, +1 is plain-HTTP forward, +2 is management _MGMT_RELOAD_TIMEOUT = 15 @@ -74,12 +71,11 @@ _BEARER_PROVIDERS: Dict[str, Tuple[str, ...]] = { "NOUS_API_KEY": ("inference.nousresearch.com",), } -# Providers authenticating with a non-Authorization header (v0.39 ``match_headers`` is -# case-insensitive). ``aliases`` are interchangeable env names for the SAME credential and -# MUST collapse into one mapping: every rule is ``require: true`` and two require-rules on one -# host would reject each other's requests. The sandbox gets the token under every name. -# Authorization is also matched for Anthropic/Azure so an SDK sending the token as Bearer still -# swaps; Gemini's ``?key=`` SDK style is covered by match_query. +# Non-Authorization-header providers (v0.39 ``match_headers`` is case-insensitive). ``aliases`` +# name the SAME credential and MUST collapse into one mapping: every rule is ``require: true`` and +# two require-rules on one host would reject each other's requests; the sandbox gets the token +# under every name. Authorization is also matched for Anthropic/Azure (SDKs may send Bearer); +# Gemini's ``?key=`` style is covered by match_query. _HEADER_AUTH_PROVIDERS: Dict[str, Dict[str, Tuple[str, ...]]] = { "ANTHROPIC_API_KEY": {"hosts": ("api.anthropic.com",), "match_headers": ("x-api-key", "Authorization"), "aliases": ()}, "AZURE_OPENAI_API_KEY": { @@ -94,14 +90,13 @@ _HEADER_AUTH_PROVIDERS: Dict[str, Dict[str, Tuple[str, ...]]] = { }, } -# Recognized creds that cannot be swapped by static header replacement (SigV4 signing, -# SDK-minted OAuth). Surfaced as a warning only — they're generic cloud creds, never blocking. +# Creds that static header replacement can't swap (SigV4, SDK-minted OAuth): warning only. _NON_BEARER_PROVIDERS: Tuple[str, ...] = ( "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "GOOGLE_APPLICATION_CREDENTIALS", ) -# Default SSRF deny list for outbound traffic (docs promise: cloud metadata IPs refused -# regardless of allowlist). Callers may pass [] to disable (hermetic tests only). +# Default SSRF deny list (docs promise: cloud metadata IPs refused regardless of allowlist); +# callers pass [] to disable (hermetic tests only). _DEFAULT_UPSTREAM_DENY_CIDRS: Tuple[str, ...] = ( "127.0.0.0/8", "::1/128", # loopback v4 / v6 "169.254.0.0/16", "fe80::/10", # link-local incl. AWS/GCP/Azure IMDS @@ -111,8 +106,7 @@ _DEFAULT_UPSTREAM_DENY_CIDRS: Tuple[str, ...] = ( "198.18.0.0/15", # RFC2544 benchmark range ) -# Minimal env the iron-proxy subprocess needs; everything else is stripped so -# /proc//environ never exposes the operator's shell secrets. +# Minimal daemon env; everything else is stripped so /proc//environ never exposes operator secrets. _PROXY_SUBPROCESS_ENV_ALLOWLIST: Tuple[str, ...] = ( "PATH", "HOME", "TMPDIR", "TZ", "LANG", "LC_ALL", "LC_CTYPE", "NO_COLOR", "SSL_CERT_DIR", "SSL_CERT_FILE", @@ -129,11 +123,11 @@ _KILL_SIGNAL = getattr(signal, "SIGKILL", signal.SIGTERM) # O_NOFOLLOW is POSIX-only; 0 is a no-op flag elsewhere. _O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0) -# ``iron-proxy --version`` output keyed by binary path (get_status runs per container create). +# ``--version`` output keyed by binary path (get_status runs per container create). _VERSION_CACHE: Dict[str, str] = {} -# Nonce planted in the daemon env at start so ``_pid_alive`` can prove a PID is still *our* -# binary across PID recycling (a fresh process can't inherit our arbitrary env value). +# Nonce planted in the daemon env so ``_pid_alive`` can prove a PID is still *our* binary across +# PID recycling (a fresh process can't inherit our arbitrary env value). _HERMES_IRON_PROXY_NONCE_ENV = "HERMES_IRON_PROXY_NONCE" _proxy_nonce: Optional[str] = None @@ -162,9 +156,8 @@ class ProxyStatus: @dataclass class TokenMapping: - """Map a sandbox-visible proxy token to a real upstream credential lookup. ``real_env_name`` - is read from iron-proxy's OWN env at egress time; ``alias_env_names`` are extra names the - SANDBOX receives the same token under (not emitted in the proxy config).""" + """Sandbox-visible proxy token -> upstream credential lookup. ``real_env_name`` is read from + iron-proxy's OWN env at egress; ``alias_env_names`` are extra SANDBOX names for the same token.""" proxy_token: str real_env_name: str upstream_hosts: Tuple[str, ...] @@ -184,8 +177,8 @@ def _proxy_state_dir_ro() -> Path: def _proxy_state_dir() -> Path: - """Proxy state dir, created 0o700 (holds the CA key, pidfile, logs); chmod is unconditional - so a pre-existing dir with a slack umask gets tightened.""" + """Proxy state dir (CA key, pidfile, logs), created 0o700; chmod is unconditional so a + pre-existing slack-umask dir gets tightened.""" d = _proxy_state_dir_ro() d.mkdir(parents=True, exist_ok=True) with suppress(OSError): # Windows no-op / shared fs we don't own; files still get explicit perms @@ -236,7 +229,6 @@ def install_iron_proxy(*, force: bool = False) -> Path: target = bin_dir / _platform_binary_name() if target.exists() and not force: return target - asset_name = _platform_asset_name() with tempfile.TemporaryDirectory(prefix="hermes-iron-proxy-") as tmpdir: tmp = Path(tmpdir) @@ -251,24 +243,20 @@ def install_iron_proxy(*, force: bool = False) -> Path: actual = _sha256_file(archive_path) if expected.lower() != actual.lower(): raise RuntimeError(f"Checksum mismatch for {asset_name}: expected {expected}, got {actual}") - with tarfile.open(archive_path, "r:gz") as tf: member = _pick_tar_member(tf, _platform_binary_name()) - # PEP 706 data filter rejects escaping links; Python < 3.12 lacks the kwarg and - # relies on _pick_tar_member's path sanitization. + # PEP 706 data filter rejects escaping links; < 3.12 relies on _pick_tar_member's sanitization. try: tf.extract(member, tmp, filter="data") # noqa: S202 except TypeError: tf.extract(member, tmp) # noqa: S202 extracted = tmp / member.name - # Stage then atomically rename so the binary is never visible half-written. fd, staged = tempfile.mkstemp(dir=str(bin_dir), prefix=".iron-proxy_") os.close(fd) shutil.copy2(extracted, staged) os.chmod(staged, 0o755) os.replace(staged, target) - # A freshly-installed binary must re-probe --version on the next get_status(). _VERSION_CACHE.pop(str(target), None) logger.info("Installed iron-proxy %s at %s", _IRON_PROXY_VERSION, target) @@ -287,14 +275,13 @@ def _release_asset(name: str, dest: Path) -> None: def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: - """Best-effort GPG verification of checksums.txt in an ephemeral keyring. Returns False (with - a warning) when gpg or the signature assets are unavailable — SHA-256 stays enforced and gpg - is never a hard dependency. Raises ONLY on a present-but-bad signature (tamper signal).""" + """Best-effort GPG check of checksums.txt in an ephemeral keyring. False (with a warning) when + gpg or the signature assets are unavailable — SHA-256 stays enforced, gpg is never a hard + dependency. Raises ONLY on a present-but-bad signature (tamper signal).""" gpg = shutil.which("gpg") if not gpg: logger.warning("gpg not found on PATH — skipping iron-proxy release-signature verification (SHA-256 checksum check still enforced).") return False - sig_path, pubkey_path = tmp / _IRON_PROXY_CHECKSUM_SIG_NAME, tmp / _IRON_PROXY_PUBKEY_NAME try: _release_asset(_IRON_PROXY_CHECKSUM_SIG_NAME, sig_path) @@ -302,7 +289,6 @@ def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: except RuntimeError as exc: logger.warning("iron-proxy release signature assets unavailable (%s) — skipping GPG verification (SHA-256 checksum check still enforced).", exc) return False - gnupg_home = tmp / "gnupg" gnupg_home.mkdir(mode=0o700, exist_ok=True) gpg_base = [gpg, "--homedir", str(gnupg_home), "--batch", "--no-tty"] @@ -341,8 +327,7 @@ def _pick_tar_member(tf: tarfile.TarFile, binary_name: str) -> tarfile.TarInfo: """Find the binary in the archive (flat or one dir deep); reject abs paths and ``..``.""" candidates = [ m for m in tf.getmembers() - if m.isfile() and not m.name.startswith("/") and ".." not in Path(m.name).parts - and Path(m.name).name == binary_name + if m.isfile() and not m.name.startswith("/") and ".." not in Path(m.name).parts and Path(m.name).name == binary_name ] if not candidates: raise RuntimeError(f"Could not find {binary_name} inside downloaded archive (members: {[m.name for m in tf.getmembers()[:5]]}...)") @@ -378,8 +363,8 @@ def iron_proxy_version(binary: Path) -> str: def _write_private_file(path: Path, data: bytes) -> None: - """Create/truncate ``path`` 0o600 from the first byte (no chmod-after TOCTOU); O_NOFOLLOW - defends against a planted symlink; fchmod tightens a pre-existing file.""" + """Create/truncate ``path`` 0o600 from the first byte (no chmod-after TOCTOU), O_NOFOLLOW + against a planted symlink, fchmod to tighten a pre-existing file.""" fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC | _O_NOFOLLOW, 0o600) try: with suppress(OSError, AttributeError): @@ -406,17 +391,14 @@ def ensure_ca_cert(*, force: bool = False) -> Tuple[Path, Path]: return ca_crt, ca_key if shutil.which("openssl") is None: raise RuntimeError("openssl not found on PATH. Install OpenSSL (apt: `openssl`, brew: `openssl`) to generate the iron-proxy CA cert.") - with tempfile.TemporaryDirectory(prefix="hermes-proxy-ca-") as tmpdir: tmp_key, tmp_crt = Path(tmpdir) / "ca.key", Path(tmpdir) / "ca.crt" _run(["openssl", "genrsa", "-out", str(tmp_key), "4096"], timeout=60, check=True) _run([ - "openssl", "req", "-x509", "-new", "-nodes", "-key", str(tmp_key), - "-sha256", "-days", "3650", "-subj", "/CN=hermes iron-proxy CA", - "-addext", "basicConstraints=critical,CA:TRUE", "-addext", "keyUsage=critical,keyCertSign", - "-out", str(tmp_crt), + "openssl", "req", "-x509", "-new", "-nodes", "-key", str(tmp_key), "-sha256", "-days", "3650", + "-subj", "/CN=hermes iron-proxy CA", "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign", "-out", str(tmp_crt), ], timeout=60, check=True) - # Key: stage 0o600 against a fresh inode, then atomically rename into place. key_staged = ca_key.with_suffix(ca_key.suffix + ".staged") key_staged.unlink(missing_ok=True) @@ -425,7 +407,6 @@ def ensure_ca_cert(*, force: bool = False) -> Tuple[Path, Path]: # Cert is public — 0o644 matches typical PEM layout. ca_crt.write_bytes(tmp_crt.read_bytes()) os.chmod(ca_crt, 0o644) - logger.info("Generated iron-proxy CA at %s", ca_crt) return ca_crt, ca_key @@ -505,8 +486,8 @@ _RELOAD_HTTP_ERRORS = { def reload_proxy() -> bool: - """Hot-reload the ruleset via ``POST /v1/reload`` (validation failures leave the running - config untouched). Raises RuntimeError with an actionable message on any failure.""" + """``POST /v1/reload`` (validation failures leave the running config untouched); RuntimeError + with an actionable message on any failure.""" pid = _read_pid() if not pid or not _pid_alive(pid): raise RuntimeError("iron-proxy is not running — nothing to reload. Run `hermes egress start`.") @@ -519,11 +500,8 @@ def reload_proxy() -> bool: token = _read_text_or_none(_proxy_state_dir_ro() / "management.token") if not token: raise RuntimeError("management.token is missing — re-run `hermes egress setup`, then `hermes egress restart`.") - host, port = mgmt - req = urllib.request.Request( - f"http://{host}:{port}/v1/reload", method="POST", headers={"Authorization": f"Bearer {token}"}, data=b"", - ) + req = urllib.request.Request(f"http://{host}:{port}/v1/reload", method="POST", headers={"Authorization": f"Bearer {token}"}, data=b"") try: with urllib.request.urlopen(req, timeout=_MGMT_RELOAD_TIMEOUT) as resp: if resp.status == 200: @@ -544,10 +522,9 @@ def reload_proxy() -> bool: def _default_http_listen(tunnel_port: int) -> List[str]: - """Single bind (v0.39 allows one per daemon): docker bridge gateway on Linux — that's what - ``host.docker.internal`` resolves to there, loopback is unreachable from containers — and - loopback on macOS/Windows Docker Desktop (VPNkit). NEVER 0.0.0.0: a LAN peer with a leaked - sandbox token could spend the operator's API quota.""" + """Single bind (v0.39 allows one): docker bridge on Linux (what ``host.docker.internal`` + resolves to; loopback is unreachable from containers), loopback on Docker Desktop (VPNkit). + NEVER 0.0.0.0: a LAN peer with a leaked sandbox token could spend the operator's API quota.""" if platform.system() == "Linux": bridge_ip = _detect_docker_bridge_ip() if bridge_ip and bridge_ip != "127.0.0.1": @@ -561,8 +538,7 @@ def _default_http_listen(tunnel_port: int) -> List[str]: def _detect_docker_bridge_ip() -> Optional[str]: """docker0 IPv4 via ``ip -4 addr show docker0``, or None. SECURITY: validated through - :mod:`ipaddress` so a hostile ``ip`` shim on PATH can't inject ``0.0.0.0`` (or - loopback/multicast/link-local/public) and reopen INADDR_ANY binding.""" + :mod:`ipaddress` so a hostile ``ip`` shim can't inject 0.0.0.0/loopback/multicast/link-local/public.""" try: res = _run(["ip", "-4", "-o", "addr", "show", "docker0"], timeout=2, text=True) except (OSError, subprocess.TimeoutExpired): @@ -592,20 +568,18 @@ def build_proxy_config( audit_log: Optional[Path] = None, allowed_hosts: Optional[List[str]] = None, upstream_deny_cidrs: Optional[List[str]] = None, http_listen: Optional[List[str]] = None, ) -> Dict: - """Build the iron-proxy YAML config dict (schema as of v0.39.0). Real secrets are read from - iron-proxy's OWN env (``source: {type: env}``); the sandbox never sees them. - ``upstream_deny_cidrs=None`` means the default SSRF deny list; ``[]`` opts out. ``audit_log`` - is accepted for forward-compat but unused: v0.39's ``config.Log`` rejects ``audit_path``.""" + """iron-proxy YAML config dict (v0.39.0 schema). Real secrets come from iron-proxy's OWN env + (``source: {type: env}``); the sandbox never sees them. ``upstream_deny_cidrs=None`` = default + SSRF deny list, ``[]`` opts out. ``audit_log`` is forward-compat only (v0.39 rejects ``audit_path``).""" hosts: List[str] = list(allowed_hosts or _DEFAULT_ALLOWED_HOSTS) for m in mappings: for h in m.upstream_hosts: if h not in hosts: hosts.append(h) deny_cidrs = list(_DEFAULT_UPSTREAM_DENY_CIDRS if upstream_deny_cidrs is None else upstream_deny_cidrs) - - # Per-provider headers (case-insensitive in v0.39); query scan covers ``?key=`` SDKs; - # body inspection deliberately off. ``require`` fails closed: an allowlisted-host request - # WITHOUT the proxy token is rejected, so a real key sent directly can't cross the boundary. + # Query scan covers ``?key=`` SDKs; body inspection deliberately off. ``require`` fails + # closed: an allowlisted-host request WITHOUT the proxy token is rejected, so a real key sent + # directly can't cross the boundary. secrets_rules = [ { "source": {"type": "env", "var": m.real_env_name}, @@ -617,29 +591,23 @@ def build_proxy_config( } for m in mappings ] - - # v0.39 takes ONE string per listener field (no plural form). tunnel_listen is the - # CONNECT+MITM listener sandboxes must reach via HTTPS_PROXY (a CONNECT to http_listen is - # forwarded upstream and 400s); http_listen is plain-HTTP forward on tunnel_port+1. + # ONE string per listener field. tunnel_listen is the CONNECT+MITM listener sandboxes reach via + # HTTPS_PROXY (a CONNECT to http_listen is forwarded upstream and 400s); http_listen is plain-HTTP + # forward on tunnel_port+1. listens = list(http_listen) if http_listen else _default_http_listen(tunnel_port) primary_listen = listens[0] if listens else f"127.0.0.1:{tunnel_port}" bind_host = primary_listen.rsplit(":", 1)[0] or "127.0.0.1" - return { # Required by the parser; tunnel-only mode never binds an exposed DNS port. "dns": {"listen": "127.0.0.1:0", "proxy_ip": "127.0.0.1"}, "proxy": { # Both bind the docker bridge on Linux / loopback on Docker Desktop — NEVER 0.0.0.0. - "tunnel_listen": primary_listen, - "http_listen": f"{bind_host}:{tunnel_port + 1}", + "tunnel_listen": primary_listen, "http_listen": f"{bind_host}:{tunnel_port + 1}", "https_listen": "127.0.0.1:0", # direct-TLS listener is not exposed - "max_request_body_bytes": 16 * 1024 * 1024, - "max_response_body_bytes": 0, - "upstream_response_header_timeout": "120s", - "upstream_deny_cidrs": deny_cidrs, + "max_request_body_bytes": 16 * 1024 * 1024, "max_response_body_bytes": 0, + "upstream_response_header_timeout": "120s", "upstream_deny_cidrs": deny_cidrs, }, - # v0.39 defaults its metrics server to :9090 — the same as our default tunnel_port — - # so pin it to an ephemeral loopback port (metrics effectively undiscoverable). + # v0.39 defaults metrics to :9090 (our default tunnel_port) — pin to an ephemeral loopback port. "metrics": {"listen": "127.0.0.1:0"}, # Loopback only: sandboxes must never reach the management surface. "management": {"listen": f"127.0.0.1:{tunnel_port + _MGMT_PORT_OFFSET}", "api_key_env": _MGMT_API_KEY_ENV}, @@ -653,9 +621,8 @@ def build_proxy_config( def _open_private_append(path: Path, *, strict_chmod: bool) -> int: - """Open ``path`` O_APPEND|O_CREAT 0o600 with O_NOFOLLOW (planted symlinks refused) and tighten - a pre-existing file via fchmod (its failure is fatal iff ``strict_chmod``). Raises OSError; - caller owns the fd.""" + """Open ``path`` O_APPEND|O_CREAT 0o600 with O_NOFOLLOW (planted symlinks refused); fchmod + tightens a pre-existing file (failure fatal iff ``strict_chmod``). Raises OSError; caller owns the fd.""" fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_APPEND | _O_NOFOLLOW, 0o600) try: os.fchmod(fd, 0o600) @@ -667,8 +634,8 @@ def _open_private_append(path: Path, *, strict_chmod: bool) -> int: def ensure_audit_log(audit_path: Path) -> None: - """Pre-create the audit log 0o600 (forward-compat: v0.39 never writes it, no ``log.audit_path``). - Raises RuntimeError on any OSError (planted symlink, immutable dir, full disk).""" + """Pre-create the audit log 0o600 (forward-compat: v0.39 never writes it). RuntimeError on any + OSError (planted symlink, immutable dir, full disk).""" try: os.close(_open_private_append(audit_path, strict_chmod=True)) except OSError as exc: @@ -679,8 +646,8 @@ def ensure_audit_log(audit_path: Path) -> None: def _write_state_file_atomic(state: Path, name: str, dump) -> Path: - """Write ``/`` via a 0600 temp file + atomic replace: the file holds proxy - token values, and chmod-after-replace would leave a world-readable TOCTOU window.""" + """0600 temp file + atomic replace: the file holds proxy tokens, and chmod-after-replace would + leave a world-readable TOCTOU window.""" tmp_path = state / f".{name}.tmp" with open(tmp_path, "w", encoding="utf-8") as f: dump(f) @@ -744,9 +711,8 @@ def _env_names(available_env_names: Optional[List[str]]) -> set: def discover_provider_mappings(*, available_env_names: Optional[List[str]] = None) -> List[TokenMapping]: - """Mint a TokenMapping for every known provider whose env var is set (bearer providers first). - Canonical OR any alias present -> ONE mapping keyed on the canonical name (the subprocess-env - builder mirrors an alias value into it).""" + """One TokenMapping per known provider whose env var is set (bearer providers first). Canonical + OR any alias present -> ONE mapping on the canonical name (the subprocess-env builder mirrors aliases).""" names = _env_names(available_env_names) specs = [(n, h, ("Authorization",), ()) for n, h in _BEARER_PROVIDERS.items()] + [ (n, tuple(s["hosts"]), tuple(s["match_headers"]), tuple(s.get("aliases") or ())) @@ -766,9 +732,8 @@ def discover_uncovered_providers(*, available_env_names: Optional[List[str]] = N def merge_mappings(*, existing: List[TokenMapping], discovered: List[TokenMapping], rotate: bool = False) -> List[TokenMapping]: - """Combine existing and freshly discovered mappings. Tokens already in ``existing`` are - preserved (containers baked with them keep working) while hosts/headers/aliases are refreshed - from ``discovered``; ``rotate=True`` re-mints everything; undiscovered providers are dropped.""" + """Tokens already in ``existing`` are preserved (containers baked with them keep working) while + hosts/headers/aliases refresh from ``discovered``; ``rotate=True`` re-mints; undiscovered providers drop.""" by_name = {} if rotate else {m.real_env_name: m for m in existing} return [ replace(d, proxy_token=by_name[d.real_env_name].proxy_token) if d.real_env_name in by_name else d @@ -807,8 +772,8 @@ def _persisted_nonce_path() -> Path: def _read_persisted_nonce() -> Optional[str]: - """Nonce from disk, or None if missing/unreadable/empty/not owned by us (callers fall back - to argv0 matching). O_NOFOLLOW: this read decides whether stop_proxy SIGKILLs a PID.""" + """Nonce from disk, or None if missing/unreadable/empty/not owned by us (callers fall back to + argv0 matching). O_NOFOLLOW: this read decides whether stop_proxy SIGKILLs a PID.""" try: fd = os.open(str(_persisted_nonce_path()), os.O_RDONLY | _O_NOFOLLOW) except OSError: @@ -822,9 +787,9 @@ def _read_persisted_nonce() -> Optional[str]: def _pid_alive(pid: int) -> bool: - """True iff ``pid`` is alive AND is an iron-proxy process. PID-reuse defense, in priority - order: nonce in /proc//environ, argv[0] basename in /proc//cmdline, ``ps -o comm=`` - basename. A loose ``"iron-proxy" in cmdline`` match would hit ``tail iron-proxy.log``.""" + """True iff ``pid`` is alive AND an iron-proxy process. PID-reuse defense, in priority order: + nonce in /proc//environ, argv[0] basename in /proc//cmdline, ``ps -o comm=`` basename + (a loose ``"iron-proxy" in cmdline`` would hit ``tail iron-proxy.log``).""" if pid <= 0: return False try: @@ -838,7 +803,6 @@ def _pid_alive(pid: int) -> bool: os.kill(pid, 0) # windows-footgun: ok — POSIX-only branch except (ProcessLookupError, PermissionError, OSError): return False - # Strong proof: nonce from this process's start_proxy and/or the on-disk sibling file. nonce_candidates = [n for n in dict.fromkeys((_proxy_nonce, _read_persisted_nonce())) if n] if nonce_candidates: @@ -864,10 +828,8 @@ def start_proxy( install_if_missing: bool = True, refresh_secrets_from_bitwarden: bool = False, bitwarden_config: Optional[Dict] = None, ) -> ProxyStatus: """Spawn iron-proxy as a managed background subprocess (idempotent if already running). - ``refresh_secrets_from_bitwarden=True`` re-fetches upstream secrets from BWS at startup — - the rotation promise of ``credential_source: bitwarden``.""" + ``refresh_secrets_from_bitwarden`` re-fetches secrets from BWS — the ``credential_source: bitwarden`` rotation promise.""" global _proxy_nonce - existing = _read_pid() if existing and _pid_alive(existing): return get_status() @@ -877,7 +839,6 @@ def start_proxy( cfg = config_path or (_proxy_state_dir() / "proxy.yaml") if not cfg.exists(): raise RuntimeError(f"iron-proxy config not found at {cfg}. Run `hermes egress setup` first.") - # Minimal env: os.environ.copy() would expose every operator secret via /proc//environ. env = _build_proxy_subprocess_env( extra_env=extra_env, refresh_from_bitwarden=refresh_secrets_from_bitwarden, bitwarden_config=bitwarden_config, @@ -888,12 +849,9 @@ def start_proxy( # Per-start nonce for PID-recycling defense; module-global is fine (one proxy per process). _proxy_nonce = hashlib.sha256(os.urandom(16)).hexdigest() env[_HERMES_IRON_PROXY_NONCE_ENV] = _proxy_nonce - log_path = _proxy_state_dir() / "iron-proxy.log" proc = _spawn_daemon(bin_path, cfg, env, log_path) - - # Pidfile BEFORE the listening poll: if the parent dies mid-poll, `hermes egress stop` - # can still clean up the orphan. + # Pidfile BEFORE the listening poll so `hermes egress stop` can clean an orphan if the parent dies mid-poll. pidfile = _pidfile() try: _write_pidfile_safely(pidfile, proc.pid) @@ -916,8 +874,7 @@ def start_proxy( pidfile.unlink(missing_ok=True) raise KeyboardInterrupt() - # Poll with timeout (the Go binary is up in <200ms). Probe the CONFIGURED bind host — on - # Linux that's the docker bridge, where loopback never connects. + # Probe the CONFIGURED bind host (on Linux the docker bridge, where loopback never connects). probe_host, tunnel_port = _probe_target() install_handlers = platform.system() != "Windows" and threading.current_thread() is threading.main_thread() prev_sigint = prev_sigterm = None @@ -930,22 +887,20 @@ def start_proxy( if install_handlers: signal.signal(signal.SIGINT, prev_sigint) signal.signal(signal.SIGTERM, prev_sigterm) - # Process may have died right at deadline. if proc.poll() is not None: raise _exited_error() # Alive-but-not-listening is a failure: an orphan holding the port breaks every restart. if not listening: raise _abort(f"iron-proxy did not bind {probe_host}:{tunnel_port} within {_STARTUP_GRACE_SECONDS}s. Process was killed. ", kill=True) - logger.info("Started iron-proxy pid=%s config=%s", proc.pid, cfg) return get_status() def _spawn_daemon(bin_path: Path, cfg: Path, env: Dict[str, str], log_path: Path) -> "subprocess.Popen": - """Popen the daemon with stdout/stderr appended to ``log_path`` (0o600 from the first byte, - O_NOFOLLOW so a planted symlink e.g. to authorized_keys can't receive output, owner-checked). - Our log fd is closed right after Popen — the child has its dup.""" + """Popen with stdout/stderr appended to ``log_path`` (0o600 from the first byte, O_NOFOLLOW so a + planted symlink e.g. to authorized_keys can't receive output, owner-checked). Our log fd is + closed right after Popen — the child has its dup.""" try: log_fd = _open_private_append(log_path, strict_chmod=False) except OSError as exc: @@ -954,7 +909,6 @@ def _spawn_daemon(bin_path: Path, cfg: Path, env: Dict[str, str], log_path: Path st = os.fstat(log_fd) os.close(log_fd) raise RuntimeError(f"iron-proxy log {log_path} has unexpected owner uid={st.st_uid}; refusing to write.") - try: # start_new_session is POSIX-only (Windows isn't supported anyway — no upstream binary). popen_kwargs: Dict = dict(env=env, stdin=subprocess.DEVNULL, stdout=log_fd, stderr=subprocess.STDOUT) @@ -969,8 +923,8 @@ def _spawn_daemon(bin_path: Path, cfg: Path, env: Dict[str, str], log_path: Path def _await_listening(proc: "subprocess.Popen", host: str, port: int, *, on_exit) -> bool: - """Poll until ``host:port`` accepts or the grace window lapses (do-while: checks at least - once even with a 0s window). Raises ``on_exit()`` if the child dies meanwhile.""" + """Poll until ``host:port`` accepts or the grace window lapses (do-while: at least one check + even with a 0s window). Raises ``on_exit()`` if the child dies meanwhile.""" deadline = time.time() + _STARTUP_GRACE_SECONDS while True: if proc.poll() is not None: @@ -983,9 +937,8 @@ def _await_listening(proc: "subprocess.Popen", host: str, port: int, *, on_exit) def _write_pidfile_safely(pidfile: Path, pid: int) -> None: - """Write the pidfile with O_EXCL + O_NOFOLLOW + ownership check, then persist the nonce. - O_EXCL on an existing pidfile means either a concurrent start (fail cleanly) or a stale - crash leftover (unlink and retry once).""" + """O_EXCL + O_NOFOLLOW + ownership check, then persist the nonce. An existing pidfile is either + a concurrent start (fail cleanly) or a stale crash leftover (unlink and retry once).""" open_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_NOFOLLOW try: fd = os.open(str(pidfile), open_flags, 0o600) @@ -998,14 +951,12 @@ def _write_pidfile_safely(pidfile: Path, pid: int) -> None: except OSError as exc: # ELOOP from a planted symlink at the pidfile path. raise RuntimeError(f"Refusing to write pidfile {pidfile}: {exc}. Remove that path manually and retry.") from exc - try: if not _fd_owned_by_us(fd): raise RuntimeError(f"pidfile {pidfile} has unexpected owner uid={os.fstat(fd).st_uid}") os.write(fd, str(pid).encode("utf-8")) finally: os.close(fd) - # Best-effort nonce sibling (0o600); without it stop falls back to argv0 matching. if _proxy_nonce: with suppress(OSError): @@ -1030,15 +981,12 @@ def _kill_and_wait(proc: "subprocess.Popen", *, grace_seconds: int = 2) -> None: def _build_proxy_subprocess_env( *, extra_env: Optional[Dict[str, str]] = None, refresh_from_bitwarden: bool = False, bitwarden_config: Optional[Dict] = None, ) -> Dict[str, str]: - """Minimal env for the daemon: allowlisted infra vars + the secrets named in mappings. With - ``refresh_from_bitwarden`` and a populated ``bitwarden_config``, secrets are fetched from BWS - at startup (the rotation guarantee); without ``allow_env_fallback`` any BWS shortfall fails - closed rather than silently keeping stale host-env values.""" + """Allowlisted infra vars + the secrets named in mappings. With ``refresh_from_bitwarden`` and a + populated ``bitwarden_config`` secrets come from BWS (the rotation guarantee); without + ``allow_env_fallback`` any BWS shortfall fails closed instead of keeping stale host-env values.""" env = _allowlisted_env() parent = os.environ - - # Forward ONLY the secrets named in mappings. For alias providers the rule is keyed on the - # canonical name; mirror the alias value into it when only the alias is set. + # Forward ONLY mapped secrets; the rule is keyed on the canonical name, so mirror an alias value into it. mappings = load_mappings() needed = {m.real_env_name for m in mappings} alias_sources = {m.real_env_name: m.alias_env_names for m in mappings if m.alias_env_names} @@ -1046,10 +994,8 @@ def _build_proxy_subprocess_env( source = name if name in parent else next((a for a in alias_sources.get(name, ()) if parent.get(a)), None) if source is not None: env[name] = parent[source] - if refresh_from_bitwarden and bitwarden_config: _refresh_secrets_from_bitwarden(env, needed, bitwarden_config, bool(bitwarden_config.get("allow_env_fallback"))) - # Caller overrides win (wizard test secrets), then strip proxy-recursion vars regardless. if extra_env: env.update(extra_env) @@ -1060,16 +1006,15 @@ def _build_proxy_subprocess_env( def _bitwarden_shortfall(allow_env_fallback: bool, error: str, warning: str, *args) -> None: - """Fail closed with ``error`` unless the operator opted into the legacy host-env fallback, - in which case only ``warning`` (``%``-formatted with ``args``) is logged.""" + """Raise ``error`` unless the operator opted into the legacy host-env fallback (then log ``warning``).""" if not allow_env_fallback: raise RuntimeError(error) logger.warning(warning, *args) def _refresh_secrets_from_bitwarden(env: Dict[str, str], needed: set, bitwarden_config: Dict, allow_env_fallback: bool) -> None: - """Overwrite ``env[needed]`` with fresh BWS values (uncached fetch). Only mapped names are - injected so unrelated BWS secrets never leak into the daemon env.""" + """Overwrite ``env[needed]`` with fresh (uncached) BWS values; only mapped names are injected so + unrelated BWS secrets never leak into the daemon env.""" try: # Lazy: the bitwarden module isn't importable in every install. from agent.secret_sources import bitwarden as bw @@ -1095,7 +1040,6 @@ def _refresh_secrets_from_bitwarden(env: Dict[str, str], needed: set, bitwarden_ ) from exc logger.warning("Bitwarden refresh module unavailable at proxy start, falling back to parent env (allow_env_fallback=true): %s", exc) return - missing = sorted(needed - set(secrets)) for n in needed: if n in secrets: @@ -1128,7 +1072,6 @@ def stop_proxy() -> bool: if not pid or not _pid_alive(pid): _forget_daemon() return False - # Capture starttime BEFORE signalling: if the pid is recycled mid-wait, abort the SIGKILL. starttime_before = _pid_proc_starttime(pid) try: @@ -1136,7 +1079,6 @@ def stop_proxy() -> bool: except ProcessLookupError: _forget_daemon() return False - # Up to 5s for graceful exit, then SIGKILL — unless the pid was recycled meanwhile. deadline = time.time() + 5.0 while time.time() < deadline: @@ -1151,7 +1093,6 @@ def stop_proxy() -> bool: else: with suppress(ProcessLookupError): os.kill(pid, _KILL_SIGNAL) - _forget_daemon() logger.info("Stopped iron-proxy pid=%s", pid) return True From eea8e8060eeb7db049cbce7f79a4d9d4b2df7bcb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:53:00 -0700 Subject: [PATCH 6/9] refactor(agent): compact module docstrings (outbound_webhooks, otlp_exporter, emitter, gateway_health_export) --- agent/monitoring/emitter.py | 14 ++++++------- agent/monitoring/gateway_health_export.py | 14 ++++++------- agent/monitoring/otlp_exporter.py | 12 +++++------ agent/outbound_webhooks.py | 25 ++++++++--------------- 4 files changed, 26 insertions(+), 39 deletions(-) diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py index 2858d257c6..b54ac54918 100644 --- a/agent/monitoring/emitter.py +++ b/agent/monitoring/emitter.py @@ -1,11 +1,9 @@ -"""Monitoring emitter: fire-and-forget queue + background dispatcher. - -The single seam between producers (gateway status hooks, diagnostic log handler) -and consumers (OTLP streamers). Hot-path invariant: ``emit()`` MUST return in -O(microseconds), MUST NOT block on disk/network, and MUST NEVER raise into the -caller — a monitoring failure is logged locally and dropped. On a full queue the -*oldest* event is dropped. A daemon thread fans batches out to fail-isolated -subscribers. Nothing is persisted: monitoring is an egress path, not a store. +"""Monitoring emitter: fire-and-forget queue + background dispatcher — the single seam between +producers (gateway status hooks, diagnostic log handler) and consumers (OTLP streamers). +Hot-path invariant: ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, and +MUST NEVER raise into the caller — a monitoring failure is logged locally and dropped. On a full +queue the *oldest* event is dropped. A daemon thread fans batches out to fail-isolated +subscribers. Nothing is persisted: monitoring is an egress path, not a store. """ from __future__ import annotations diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 643298be61..68f69bccc9 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -171,11 +171,10 @@ def _count(failure_msg: str, module: str, read: Callable[[Any], Any]) -> int: def _read_background_work_count() -> int: - """Live background/subagent work that ``active_agents`` deliberately does NOT include - (``active_agents`` = foreground turns + in-flight cron + API runs; backgrounded - ``delegate_task`` subagents, ``terminal(background=true)`` processes and kanban workers are - tracked only by the scale-to-zero guard). TASK-granular: a fan-out batch of N contributes N - (real concurrent load), unlike the pool's one-slot-per-batch accounting. Content-free.""" + """Live background/subagent work that ``active_agents`` (foreground turns + in-flight cron + API + runs) deliberately does NOT include: backgrounded ``delegate_task`` subagents, + ``terminal(background=true)`` processes, kanban workers. TASK-granular: a fan-out batch of N + contributes N (real concurrent load), unlike the pool's one-slot-per-batch accounting.""" return ( _count("background-work async-delegation count failed", "tools.async_delegation", lambda m: m.active_task_count()) + _count("background-work process-registry count failed", "tools.process_registry", @@ -184,9 +183,8 @@ def _read_background_work_count() -> int: def _read_background_delegations_count() -> int: - """Live async delegation UNITS (dispatch/pool slots): a batch counts ONE regardless of fan-out - width, matching the pool's capacity accounting — slot pressure (alert vs - ``max_concurrent_children``) alongside ``background_work``'s real load. Delegations only.""" + """Live async delegation UNITS (pool slots): a batch counts ONE regardless of fan-out width, so + operators see slot pressure (vs ``max_concurrent_children``) alongside ``background_work``.""" return _count("background-delegations count failed", "tools.async_delegation", lambda m: m.active_count()) diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index 2f845569ff..e5f054db5c 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -1,12 +1,10 @@ """Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP. -Maps gateway monitoring events to OTel spans for the operator-configured -``monitoring.export.otlp`` endpoint (no default destination ships) and hosts the OTLP -plumbing shared with ``gateway_health_export`` (SDK loading, header resolution, resource -attributes, endpoint mapping). The OTel SDK is an optional extra (``hermes-agent[otlp]``) -imported lazily; ``headers_env`` values are read from the environment at export time and -never logged or stored. The continuous subscriber runs on the emitter's dispatcher thread, -fail-isolated, and ``event_filter`` keeps other planes from riding along on this exporter. +Maps gateway monitoring events to OTel spans for the operator-configured ``monitoring.export.otlp`` +endpoint (no default destination ships) and hosts the OTLP plumbing shared with +``gateway_health_export``. The OTel SDK is an optional extra (``hermes-agent[otlp]``) imported +lazily; ``headers_env`` values are read at export time and never logged or stored. The streaming +subscriber runs fail-isolated on the emitter thread; ``event_filter`` keeps other planes off it. """ from __future__ import annotations diff --git a/agent/outbound_webhooks.py b/agent/outbound_webhooks.py index 4504838888..91d430060b 100644 --- a/agent/outbound_webhooks.py +++ b/agent/outbound_webhooks.py @@ -1,14 +1,10 @@ -"""Outbound webhook notifications: ``hooks.outbound`` in config.yaml -> notify-only callbacks -on the plugin hook manager, so every ``invoke_hook()`` site can push lifecycle events to -external HTTP endpoints (outbound mirror of ``gateway/platforms/webhook.py``). - -* Fire-and-forget: callbacks serialize, enqueue on a bounded queue and return ``None``; one - daemon worker POSTs, so a target can never block a tool call or influence agent flow. -* HMAC-SHA256 signed (``X-Hermes-Signature-256: sha256=`` over the raw body) when a - secret is configured. ``HERMES_SAFE_MODE=1`` skips registration; registration is idempotent. -* Entry keys: url, events, secret_env|secret, matcher (pre/post_tool_call only), timeout - (clamped to [1, 60]), name. Body: ``{hook_event_name, profile, tool_name, tool_input, - session_id, cwd, extra, delivery_id, timestamp}``. +"""Outbound webhooks: ``hooks.outbound`` entries (url, events, secret_env|secret, matcher for +pre/post_tool_call, timeout clamped to [1, 60], name) -> notify-only callbacks on the plugin hook +manager, so every ``invoke_hook()`` site can POST lifecycle events (mirror of +``gateway/platforms/webhook.py``). Fire-and-forget through a bounded queue + one daemon worker, +so a target can never block a tool call or influence agent flow. HMAC-SHA256 signed +(``X-Hermes-Signature-256: sha256=`` over the raw body) when a secret is configured; +``HERMES_SAFE_MODE=1`` skips registration; registration is idempotent. """ from __future__ import annotations @@ -76,11 +72,8 @@ class WebhookTarget(_ToolMatcherMixin): # --- Public API ----------------------------------------------------------------- def register_from_config(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: - """Register every configured outbound webhook on the plugin manager. - - Malformed ``hooks.outbound`` means zero targets — never raises. Returns the - targets that ended up wired (deduplicated across repeat calls). - """ + """Register every configured outbound webhook on the plugin manager. Malformed ``hooks.outbound`` + means zero targets — never raises. Returns the targets that ended up wired (deduplicated).""" if not isinstance(cfg, dict): return [] from utils import env_var_enabled From af0ddcb2ec6b5a6e514c60c89ad2b82ae5a7327f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:58:26 -0700 Subject: [PATCH 7/9] refactor(agent): iron_proxy signal guard ctx manager + yaml loader helper; table/kwargs compaction across group --- agent/monitoring/cron_health.py | 16 +++--- agent/monitoring/emitter.py | 7 +-- agent/monitoring/gateway_health.py | 28 ++++------ agent/outbound_webhooks.py | 27 +++------- agent/proxy_sources/iron_proxy.py | 85 +++++++++++++++--------------- 5 files changed, 66 insertions(+), 97 deletions(-) diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py index de4e9ea73d..ecd9e739fd 100644 --- a/agent/monitoring/cron_health.py +++ b/agent/monitoring/cron_health.py @@ -37,8 +37,8 @@ class CronHealthSnapshot: _job_key = _safe_instance_id _AUTH_RE = re.compile( - r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize" - r"|unauthorized|forbidden|bearer|401|403)\b|\b(?:access|api|refresh) token\b" + r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden|bearer|401|403)\b" + r"|\b(?:access|api|refresh) token\b" ) # Ordered (predicate, class) rules; first match wins. Auth uses word boundaries so @@ -146,15 +146,11 @@ def _single_metric(name: str, reader: Callable[[], Any]) -> Callable[[list[Gatew # Each group is independently fail-open so one unavailable source never hides the rest. # Readers are wrapped in lambdas so monkeypatching this module's names still takes effect. _METRIC_GROUPS: tuple[tuple[Callable[[list[GatewayMetric]], None], str], ...] = ( - (_freshness_metric("hermes.cron.scheduler.heartbeat_age_seconds", lambda: get_ticker_heartbeat_age()), - "cron freshness metric unavailable"), - (_freshness_metric("hermes.cron.scheduler.last_success_age_seconds", lambda: get_ticker_success_age()), - "cron freshness metric unavailable"), - (_single_metric("hermes.cron.scheduler.catch_up_occurrences", lambda: get_catch_up_occurrence_count()), - "cron catch-up metric unavailable"), + (_freshness_metric("hermes.cron.scheduler.heartbeat_age_seconds", lambda: get_ticker_heartbeat_age()), "cron freshness metric unavailable"), + (_freshness_metric("hermes.cron.scheduler.last_success_age_seconds", lambda: get_ticker_success_age()), "cron freshness metric unavailable"), + (_single_metric("hermes.cron.scheduler.catch_up_occurrences", lambda: get_catch_up_occurrence_count()), "cron catch-up metric unavailable"), (_job_metrics, "cron job metrics unavailable"), - (_single_metric("hermes.cron.jobs.running", lambda: len(get_running_job_ids())), - "cron running-job metric unavailable"), + (_single_metric("hermes.cron.jobs.running", lambda: len(get_running_job_ids())), "cron running-job metric unavailable"), ) diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py index b54ac54918..d3cf44a27d 100644 --- a/agent/monitoring/emitter.py +++ b/agent/monitoring/emitter.py @@ -123,12 +123,7 @@ class MonitoringEmitter: finished.wait(timeout=timeout) def stats(self) -> Dict[str, int]: - return { - "queued": self._q.qsize(), - "dispatched": self._dispatched, - "dropped": self._dropped, - "subscribers": len(self._subscribers), - } + return {"queued": self._q.qsize(), "dispatched": self._dispatched, "dropped": self._dropped, "subscribers": len(self._subscribers)} def close(self) -> None: self._stop.set() diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index c305110896..21419a4f25 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -60,10 +60,8 @@ _GATEWAY_ERROR_RULES: tuple[tuple[Callable[[str], bool], str], ...] = ( (_contains_any("auth", "token", "unauthorized", "forbidden", "401", "403"), "auth_failed"), (lambda s: "rate" in s and "limit" in s, "rate_limited"), (_contains_any("timeout", "timed out"), "timeout"), - (_contains_any( - "network", "connection", "dns", "socket", "connect call failed", - "failed to connect", "cannot connect", "unreachable", "name resolution", - ), "network_error"), + (_contains_any("network", "connection", "dns", "socket", "connect call failed", "failed to connect", "cannot connect", + "unreachable", "name resolution"), "network_error"), (_contains_any("config", "missing", "invalid"), "invalid_config"), (_contains_any("startup"), "startup_failed"), (_contains_any("fatal"), "platform_fatal"), @@ -126,13 +124,13 @@ def platform_for_subsystem(subsystem: str) -> Optional[str]: return (subsystem.split(".", 1)[1] or None) if subsystem.startswith("platform.") else None -def _gateway_status(name: str, fallback: Callable[..., Any], /, **kwargs: Any) -> Any: +def _gateway_status(name: str, fallback: Callable[[], Any], /, **kwargs: Any) -> Any: """Prefer ``gateway.status.`` (the runtime-status contract); fall back to the local approximation.""" try: import gateway.status as status return getattr(status, name)(**kwargs) except Exception: - return fallback(**kwargs) + return fallback() def _int_or_zero(raw: Any) -> int: @@ -143,25 +141,17 @@ def _int_or_zero(raw: Any) -> int: def _parse_active_agents(raw: Any) -> int: - return _gateway_status("parse_active_agents", _int_or_zero, raw=raw) + return _gateway_status("parse_active_agents", lambda: _int_or_zero(raw), raw=raw) def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: - return _gateway_status( - "derive_gateway_busy", - lambda gateway_running, gateway_state, active_agents: bool( - gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0 - ), - gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents, - ) + fallback = lambda: bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0) # noqa: E731 + return _gateway_status("derive_gateway_busy", fallback, gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents) def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool: - return _gateway_status( - "derive_gateway_drainable", - lambda gateway_running, gateway_state: bool(gateway_running and gateway_state == "running"), - gateway_running=gateway_running, gateway_state=gateway_state, - ) + fallback = lambda: bool(gateway_running and gateway_state == "running") # noqa: E731 + return _gateway_status("derive_gateway_drainable", fallback, gateway_running=gateway_running, gateway_state=gateway_state) def _base_attrs(*, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]: diff --git a/agent/outbound_webhooks.py b/agent/outbound_webhooks.py index 91d430060b..d9a81af1c3 100644 --- a/agent/outbound_webhooks.py +++ b/agent/outbound_webhooks.py @@ -180,11 +180,9 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: if not isinstance(events_raw, list) or not events_raw: warn(" needs a non-empty 'events' list (valid: %s)", valid_list) return None - events: List[str] = [] + events: List[str] = [ev for ev in events_raw if ev in VALID_HOOKS] for ev in events_raw: - if ev in VALID_HOOKS: - events.append(ev) - else: + if ev not in VALID_HOOKS: warn(": unknown event %r ignored (valid: %s)", ev, valid_list) if not events: warn(" has no valid events — skipped") @@ -204,12 +202,8 @@ def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: timeout = DEFAULT_TIMEOUT_SECONDS name = raw.get("name") return WebhookTarget( - url=url, - events=events, - name=name.strip() if isinstance(name, str) else "", - secret=_resolve_secret(index, raw), - matcher=matcher, - timeout=max(1, min(timeout, MAX_TIMEOUT_SECONDS)), + url=url, events=events, name=name.strip() if isinstance(name, str) else "", secret=_resolve_secret(index, raw), + matcher=matcher, timeout=max(1, min(timeout, MAX_TIMEOUT_SECONDS)), ) @@ -260,21 +254,16 @@ def _serialize_payload(event: str, kwargs: Dict[str, Any], delivery_id: str) -> # Profile resolved at fire time so a multiplexed gateway's receivers can tell which profile emitted. from hermes_cli.profiles import get_active_profile_name payload = { - "hook_event_name": event, - "profile": get_active_profile_name(), - **_payload_fields(kwargs), - "delivery_id": delivery_id, - "timestamp": _utc_now_iso(), + "hook_event_name": event, "profile": get_active_profile_name(), **_payload_fields(kwargs), + "delivery_id": delivery_id, "timestamp": _utc_now_iso(), } return json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8") def _build_delivery(event: str, target: WebhookTarget, body: bytes, delivery_id: str) -> Dict[str, Any]: headers = { - "Content-Type": "application/json", - "User-Agent": "Hermes-Agent-Outbound-Webhook", - "X-Hermes-Event": event, - "X-Hermes-Delivery": delivery_id, + "Content-Type": "application/json", "User-Agent": "Hermes-Agent-Outbound-Webhook", + "X-Hermes-Event": event, "X-Hermes-Delivery": delivery_id, } if target.secret: digest = hmac.new(target.secret.encode("utf-8"), body, hashlib.sha256).hexdigest() diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index d5cd271165..44b462eb69 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -24,7 +24,7 @@ import threading import time import urllib.error import urllib.request -from contextlib import suppress +from contextlib import contextmanager, suppress from dataclasses import dataclass, field, replace from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -61,14 +61,10 @@ _DEFAULT_ALLOWED_HOSTS: Tuple[str, ...] = ( # Provider env-var name -> upstream hosts on which the Authorization Bearer token is swapped. _BEARER_PROVIDERS: Dict[str, Tuple[str, ...]] = { - "OPENROUTER_API_KEY": ("openrouter.ai", "*.openrouter.ai"), - "OPENAI_API_KEY": ("api.openai.com",), - "GROQ_API_KEY": ("api.groq.com",), - "TOGETHER_API_KEY": ("api.together.xyz",), - "DEEPSEEK_API_KEY": ("api.deepseek.com",), - "MISTRAL_API_KEY": ("api.mistral.ai",), - "XAI_API_KEY": ("api.x.ai",), - "NOUS_API_KEY": ("inference.nousresearch.com",), + "OPENROUTER_API_KEY": ("openrouter.ai", "*.openrouter.ai"), "OPENAI_API_KEY": ("api.openai.com",), + "GROQ_API_KEY": ("api.groq.com",), "TOGETHER_API_KEY": ("api.together.xyz",), + "DEEPSEEK_API_KEY": ("api.deepseek.com",), "MISTRAL_API_KEY": ("api.mistral.ai",), + "XAI_API_KEY": ("api.x.ai",), "NOUS_API_KEY": ("inference.nousresearch.com",), } # Non-Authorization-header providers (v0.39 ``match_headers`` is case-insensitive). ``aliases`` @@ -80,20 +76,13 @@ _HEADER_AUTH_PROVIDERS: Dict[str, Dict[str, Tuple[str, ...]]] = { "ANTHROPIC_API_KEY": {"hosts": ("api.anthropic.com",), "match_headers": ("x-api-key", "Authorization"), "aliases": ()}, "AZURE_OPENAI_API_KEY": { "hosts": ("*.openai.azure.com", "*.cognitiveservices.azure.com", "*.services.ai.azure.com"), - "match_headers": ("api-key", "Authorization"), - "aliases": (), - }, - "GEMINI_API_KEY": { - "hosts": ("generativelanguage.googleapis.com",), - "match_headers": ("x-goog-api-key",), - "aliases": ("GOOGLE_API_KEY",), + "match_headers": ("api-key", "Authorization"), "aliases": (), }, + "GEMINI_API_KEY": {"hosts": ("generativelanguage.googleapis.com",), "match_headers": ("x-goog-api-key",), "aliases": ("GOOGLE_API_KEY",)}, } # Creds that static header replacement can't swap (SigV4, SDK-minted OAuth): warning only. -_NON_BEARER_PROVIDERS: Tuple[str, ...] = ( - "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "GOOGLE_APPLICATION_CREDENTIALS", -) +_NON_BEARER_PROVIDERS: Tuple[str, ...] = ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "GOOGLE_APPLICATION_CREDENTIALS") # Default SSRF deny list (docs promise: cloud metadata IPs refused regardless of allowlist); # callers pass [] to disable (hermetic tests only). @@ -108,8 +97,7 @@ _DEFAULT_UPSTREAM_DENY_CIDRS: Tuple[str, ...] = ( # Minimal daemon env; everything else is stripped so /proc//environ never exposes operator secrets. _PROXY_SUBPROCESS_ENV_ALLOWLIST: Tuple[str, ...] = ( - "PATH", "HOME", "TMPDIR", "TZ", "LANG", "LC_ALL", "LC_CTYPE", "NO_COLOR", - "SSL_CERT_DIR", "SSL_CERT_FILE", + "PATH", "HOME", "TMPDIR", "TZ", "LANG", "LC_ALL", "LC_CTYPE", "NO_COLOR", "SSL_CERT_DIR", "SSL_CERT_FILE", "SYSTEMROOT", "USERPROFILE", # Windows ) @@ -435,11 +423,19 @@ def ensure_management_token(*, force: bool = False) -> str: return token -def _load_proxy_yaml(cfg: Path): - """Parsed proxy.yaml, or ``{}`` when missing/unreadable/PyYAML absent.""" +def _yaml(): + """PyYAML module or None (it is a Hermes dep, but never a hard requirement here).""" try: import yaml + return yaml except ImportError: + return None + + +def _load_proxy_yaml(cfg: Path): + """Parsed proxy.yaml, or ``{}`` when missing/unreadable/PyYAML absent.""" + yaml = _yaml() + if yaml is None: return {} try: return yaml.safe_load(cfg.read_text(encoding="utf-8")) or {} @@ -658,13 +654,11 @@ def _write_state_file_atomic(state: Path, name: str, dump) -> Path: def write_proxy_config(config: Dict) -> Path: """Serialize the config dict to ``/proxy/proxy.yaml`` (safe_dump, no Python tags).""" - try: - import yaml # PyYAML is already a Hermes dep - except ImportError as exc: - raise RuntimeError("PyYAML is required to write the iron-proxy config but is not installed.") from exc + yaml = _yaml() + if yaml is None: + raise RuntimeError("PyYAML is required to write the iron-proxy config but is not installed.") return _write_state_file_atomic( - _proxy_state_dir(), "proxy.yaml", - lambda f: yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False), + _proxy_state_dir(), "proxy.yaml", lambda f: yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False), ) @@ -876,17 +870,8 @@ def start_proxy( # Probe the CONFIGURED bind host (on Linux the docker bridge, where loopback never connects). probe_host, tunnel_port = _probe_target() - install_handlers = platform.system() != "Windows" and threading.current_thread() is threading.main_thread() - prev_sigint = prev_sigterm = None - if install_handlers: - prev_sigint = signal.signal(signal.SIGINT, _interrupt_handler) - prev_sigterm = signal.signal(signal.SIGTERM, _interrupt_handler) - try: + with _interrupt_guard(_interrupt_handler): listening = _await_listening(proc, probe_host, tunnel_port, on_exit=_exited_error) - finally: - if install_handlers: - signal.signal(signal.SIGINT, prev_sigint) - signal.signal(signal.SIGTERM, prev_sigterm) # Process may have died right at deadline. if proc.poll() is not None: raise _exited_error() @@ -897,6 +882,20 @@ def start_proxy( return get_status() +@contextmanager +def _interrupt_guard(handler): + """Route SIGINT/SIGTERM to ``handler`` for the block (POSIX main thread only), restoring the previous handlers after.""" + if platform.system() == "Windows" or threading.current_thread() is not threading.main_thread(): + yield + return + prev = [(sig, signal.signal(sig, handler)) for sig in (signal.SIGINT, signal.SIGTERM)] + try: + yield + finally: + for sig, old in prev: + signal.signal(sig, old) + + def _spawn_daemon(bin_path: Path, cfg: Path, env: Dict[str, str], log_path: Path) -> "subprocess.Popen": """Popen with stdout/stderr appended to ``log_path`` (0o600 from the first byte, O_NOFOLLOW so a planted symlink e.g. to authorized_keys can't receive output, owner-checked). Our log fd is @@ -911,10 +910,10 @@ def _spawn_daemon(bin_path: Path, cfg: Path, env: Dict[str, str], log_path: Path raise RuntimeError(f"iron-proxy log {log_path} has unexpected owner uid={st.st_uid}; refusing to write.") try: # start_new_session is POSIX-only (Windows isn't supported anyway — no upstream binary). - popen_kwargs: Dict = dict(env=env, stdin=subprocess.DEVNULL, stdout=log_fd, stderr=subprocess.STDOUT) - if platform.system() != "Windows": - popen_kwargs["start_new_session"] = True - return subprocess.Popen([str(bin_path), "-config", str(cfg)], **popen_kwargs) # noqa: S603 + session_kw = {} if platform.system() == "Windows" else {"start_new_session": True} + return subprocess.Popen( # noqa: S603 + [str(bin_path), "-config", str(cfg)], env=env, stdin=subprocess.DEVNULL, stdout=log_fd, stderr=subprocess.STDOUT, **session_kw, + ) except OSError as exc: raise RuntimeError(f"failed to spawn iron-proxy: {exc}") from exc finally: From cee79f052ad7c7409edb7b4bf5f4f734cbc477db Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:03:40 -0700 Subject: [PATCH 8/9] =?UTF-8?q?refactor(agent):=20iron=5Fproxy=20=E2=80=94?= =?UTF-8?q?=20rewrap=20four=20over-long=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/proxy_sources/iron_proxy.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index 44b462eb69..03ff922673 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -275,14 +275,19 @@ def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: _release_asset(_IRON_PROXY_CHECKSUM_SIG_NAME, sig_path) _release_asset(_IRON_PROXY_PUBKEY_NAME, pubkey_path) except RuntimeError as exc: - logger.warning("iron-proxy release signature assets unavailable (%s) — skipping GPG verification (SHA-256 checksum check still enforced).", exc) + logger.warning( + "iron-proxy release signature assets unavailable (%s) — skipping GPG verification (SHA-256 checksum check still enforced).", exc, + ) return False gnupg_home = tmp / "gnupg" gnupg_home.mkdir(mode=0o700, exist_ok=True) gpg_base = [gpg, "--homedir", str(gnupg_home), "--batch", "--no-tty"] imp = _run([*gpg_base, "--import", str(pubkey_path)], timeout=60) if imp.returncode != 0: - logger.warning("Could not import iron-proxy signing key — skipping GPG verification (SHA-256 still enforced): %s", imp.stderr.decode("utf-8", "replace")[:200]) + logger.warning( + "Could not import iron-proxy signing key — skipping GPG verification (SHA-256 still enforced): %s", + imp.stderr.decode("utf-8", "replace")[:200], + ) return False verify = _run([*gpg_base, "--verify", str(sig_path), str(checksum_path)], timeout=60) if verify.returncode != 0: @@ -944,7 +949,10 @@ def _write_pidfile_safely(pidfile: Path, pid: int) -> None: except FileExistsError: existing_pid = _read_pid() if existing_pid and _pid_alive(existing_pid): - raise RuntimeError(f"Another iron-proxy start appears to be in progress (pidfile {pidfile} -> pid {existing_pid}). Run `hermes egress stop` if that proxy is stuck.") + raise RuntimeError( + f"Another iron-proxy start appears to be in progress (pidfile {pidfile} -> pid {existing_pid}). " + f"Run `hermes egress stop` if that proxy is stuck." + ) pidfile.unlink(missing_ok=True) fd = os.open(str(pidfile), open_flags, 0o600) except OSError as exc: @@ -1025,7 +1033,8 @@ def _refresh_secrets_from_bitwarden(env: Dict[str, str], needed: set, bitwarden_ allow_env_fallback, "credential_source=bitwarden but the access-token env or project_id is empty. Either set both, switch to " "credential_source: env, or set `proxy.allow_env_fallback: true` to opt into the legacy fallback behaviour.", - "credential_source=bitwarden but access-token env or project_id is empty — proxy will fall back to parent env (allow_env_fallback=true).", + "credential_source=bitwarden but access-token env or project_id is empty — " + "proxy will fall back to parent env (allow_env_fallback=true).", ) return secrets, warnings = bw.fetch_bitwarden_secrets(access_token=access_token, project_id=project_id, cache_ttl_seconds=0, use_cache=False) From 5e0aa62754632b7bf12c7a7037fc4bd8b60066fe Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:12:59 -0700 Subject: [PATCH 9/9] =?UTF-8?q?refactor(agent):=20iron=5Fproxy=20=E2=80=94?= =?UTF-8?q?=20keep=20=5Fread=5Fhttp=5Flisten=5Ffrom=5Fconfig=20name=20(cit?= =?UTF-8?q?ed=20in=20egress-internals=20docs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/proxy_sources/iron_proxy.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index 03ff922673..904c523025 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -470,11 +470,16 @@ def _read_management_listen_from_config(config_path: Optional[Path] = None) -> O return _config_listen("management", "listen", config_path=config_path) +def _read_http_listen_from_config() -> Optional[Tuple[str, int]]: + """Sandbox-facing listener: ``tunnel_listen`` (CONNECT/MITM), falling back to ``http_listen`` + for configs written before the listener-role split.""" + return _config_listen("proxy", "tunnel_listen", "http_listen") + + def _probe_target() -> Tuple[str, int]: """Configured bind host/port to probe — on Linux that's the docker bridge, where a loopback - connect would report a healthy daemon as down. ``tunnel_listen`` (CONNECT/MITM) wins, falling - back to ``http_listen`` for configs written before the listener-role split.""" - return _config_listen("proxy", "tunnel_listen", "http_listen") or ("127.0.0.1", _DEFAULT_TUNNEL_PORT) + connect would report a healthy daemon as down.""" + return _read_http_listen_from_config() or ("127.0.0.1", _DEFAULT_TUNNEL_PORT) # Management-API error status -> operator message (422 = validation rejected, running ruleset