refactor(cron): pack brackets, fold closers, compact docstrings in scheduler_* modules (AST-neutral)
This commit is contained in:
+251
-316
File diff suppressed because it is too large
Load Diff
+20
-34
@@ -27,19 +27,15 @@ DRIFT_SKIP_SILENT_MARKER = "[drift_skip:silent]"
|
||||
_TRANSIENT_NET_EXC_NAMES = frozenset({
|
||||
"ConnectError", "ConnectTimeout", "ReadTimeout", "WriteTimeout", "PoolTimeout", "NetworkError",
|
||||
"TimeoutException", "ClientConnectorError", "ClientConnectorDNSError", "ServerTimeoutError",
|
||||
"ClientOSError",
|
||||
})
|
||||
"ClientOSError"})
|
||||
_DNS_FAILURE_NEEDLES = ("nodename nor servname", "name or service not known")
|
||||
_TRANSIENT_OSERROR_NEEDLES = _DNS_FAILURE_NEEDLES + (
|
||||
"temporary failure in name resolution", "network is unreachable",
|
||||
)
|
||||
"temporary failure in name resolution", "network is unreachable")
|
||||
_TRANSIENT_HTTP_NEEDLES = _TRANSIENT_OSERROR_NEEDLES + (
|
||||
"failed to resolve", "connection refused", "timed out", "timeout",
|
||||
)
|
||||
"failed to resolve", "connection refused", "timed out", "timeout")
|
||||
_TRANSIENT_ERRNOS = frozenset({
|
||||
errno.ECONNREFUSED, errno.ECONNRESET, errno.EHOSTUNREACH, errno.ENETUNREACH, errno.ENETDOWN,
|
||||
errno.ETIMEDOUT, errno.EAGAIN,
|
||||
})
|
||||
errno.ETIMEDOUT, errno.EAGAIN})
|
||||
|
||||
|
||||
def _is_transient_provider_resolve_error(exc: BaseException) -> bool:
|
||||
@@ -64,8 +60,7 @@ def _is_transient_provider_resolve_error(exc: BaseException) -> bool:
|
||||
if type(cur).__name__ in _TRANSIENT_NET_EXC_NAMES:
|
||||
return True
|
||||
if any(m in module for m in ("httpx", "httpcore", "aiohttp")) and any(
|
||||
needle in msg for needle in _TRANSIENT_HTTP_NEEDLES
|
||||
):
|
||||
needle in msg for needle in _TRANSIENT_HTTP_NEEDLES):
|
||||
return True
|
||||
if isinstance(cur, OSError):
|
||||
if isinstance(cur, socket.gaierror):
|
||||
@@ -100,8 +95,7 @@ def _preflight_check_provider_key(job: dict, cfg: dict) -> Optional[str]:
|
||||
|
||||
_cron_cfg = cfg.get("cron") if isinstance(cfg.get("cron"), dict) else {}
|
||||
requested = (
|
||||
job.get("provider") or str((_cron_cfg or {}).get("model_provider") or "").strip() or None
|
||||
)
|
||||
job.get("provider") or str((_cron_cfg or {}).get("model_provider") or "").strip() or None)
|
||||
model = job.get("model") or os.getenv("HERMES_MODEL") or ""
|
||||
|
||||
from hermes_cli.auth import AuthError
|
||||
@@ -178,12 +172,10 @@ def _delivery_platform_routed_from_primary_gateway(platform_name: str) -> bool:
|
||||
|
||||
|
||||
class SharedRouteAdapters:
|
||||
"""Read-only adapter map for a credentialless satellite profile.
|
||||
|
||||
``get(platform, target)`` resolves the PRIMARY adapter iff the inbound route matcher
|
||||
(``ProfileRoute.matches``) accepts the target; anything else (unmatched target, disabled route,
|
||||
other profile, or target-less ``get(platform)``) is a miss — fail closed, never the default bot.
|
||||
"""
|
||||
"""Read-only adapter map for a credentialless satellite profile. ``get(platform, target)``
|
||||
resolves the PRIMARY adapter iff the inbound route matcher (``ProfileRoute.matches``) accepts
|
||||
the target; anything else (unmatched target, disabled route, other profile, or target-less
|
||||
``get(platform)``) is a miss — fail closed, never the default bot."""
|
||||
|
||||
def __init__(self, primary_adapters, routes) -> None:
|
||||
self._primary = dict(primary_adapters or {})
|
||||
@@ -213,17 +205,14 @@ class SharedRouteAdapters:
|
||||
|
||||
|
||||
def _preflight_check_delivery(job: dict) -> Optional[str]:
|
||||
"""Check delivery targets resolve to configured platforms.
|
||||
|
||||
``local``/``origin``/``all`` are never checked (no gateway-config load). Unknown platform always
|
||||
blocks; known platform blocks only if the gateway config loads AND reports it unconnected.
|
||||
Config load failures fail OPEN. ``failure_deliver`` is checked with the same rules: a typo'd
|
||||
failure platform would otherwise only surface when a failure occurs (NS-788).
|
||||
"""
|
||||
"""Check delivery targets resolve to configured platforms. ``local``/``origin``/``all`` are
|
||||
never checked (no gateway-config load). Unknown platform always blocks; known platform blocks
|
||||
only if the gateway config loads AND reports it unconnected; config load failures fail OPEN.
|
||||
``failure_deliver`` gets the same rules — a typo'd failure platform would otherwise only
|
||||
surface when a failure occurs (NS-788)."""
|
||||
deliver_value = _sched._normalize_deliver_value(job.get("deliver", "local"))
|
||||
failure_deliver_value = _sched._normalize_deliver_value(
|
||||
_sched._delivery_lane_value(job, for_failure=True)
|
||||
)
|
||||
_sched._delivery_lane_value(job, for_failure=True))
|
||||
lane_values = [deliver_value]
|
||||
if failure_deliver_value != deliver_value:
|
||||
lane_values.append(failure_deliver_value)
|
||||
@@ -233,7 +222,7 @@ def _preflight_check_delivery(job: dict) -> Optional[str]:
|
||||
part = part.strip()
|
||||
if not part or part.lower() in {"local", "origin", "all"}:
|
||||
continue
|
||||
# bot-chat targets deliver via a local subprocess; failures surface in last_delivery_error.
|
||||
# bot-chat targets deliver via a local subprocess; failures land in last_delivery_error.
|
||||
if _sched.parse_bot_chat_deliver_token(part) is not None:
|
||||
continue
|
||||
platform_parts.append(part.split(":", 1)[0].strip())
|
||||
@@ -258,8 +247,7 @@ def _preflight_check_delivery(job: dict) -> Optional[str]:
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"preflight: gateway config unavailable — skipping "
|
||||
"delivery credential check", exc_info=True,
|
||||
)
|
||||
"delivery credential check", exc_info=True)
|
||||
return None # fail-open
|
||||
# Multiplex: a satellite served by the primary's adapters reads unconnected — no block.
|
||||
if (
|
||||
@@ -278,8 +266,7 @@ def _preflight_check_delivery(job: dict) -> Optional[str]:
|
||||
_SKILL_MISSING_FIELDS = (
|
||||
("missing_required_environment_variables", "env ${}"),
|
||||
("missing_required_commands", "command '{}'"),
|
||||
("missing_credential_files", "credential file {}"),
|
||||
)
|
||||
("missing_credential_files", "credential file {}"))
|
||||
|
||||
|
||||
def _preflight_check_skills(job: dict) -> Optional[str]:
|
||||
@@ -322,8 +309,7 @@ def _preflight_job_config(job: dict, cfg: dict) -> Optional[str]:
|
||||
for name, check in (
|
||||
("provider_key", lambda: _preflight_check_provider_key(job, cfg)),
|
||||
("skills", lambda: _preflight_check_skills(job)),
|
||||
("delivery", lambda: _preflight_check_delivery(job)),
|
||||
):
|
||||
("delivery", lambda: _preflight_check_delivery(job))):
|
||||
try:
|
||||
reason = check()
|
||||
except Exception:
|
||||
|
||||
+38
-42
@@ -77,27 +77,30 @@ def _inject_context_from(job: dict, prompt: str) -> tuple[str, bool]:
|
||||
if not source_job_id or not all(c in "0123456789abcdef" for c in source_job_id):
|
||||
logger.warning(
|
||||
"context_from: skipping invalid job_id %r for job_id=%r name=%r%s",
|
||||
source_job_id, job.get("id"), job.get("name"), _sched._cron_job_origin_log_suffix(job),
|
||||
source_job_id, job.get("id"), job.get("name"),
|
||||
_sched._cron_job_origin_log_suffix(job),
|
||||
)
|
||||
continue
|
||||
try:
|
||||
output_files = sorted(
|
||||
(output_dir / source_job_id).glob("*.md"), key=lambda f: f.stat().st_mtime, reverse=True,
|
||||
(output_dir / source_job_id).glob("*.md"), key=lambda f: f.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not output_files:
|
||||
continue # silent skip — no output yet
|
||||
latest_output = output_files[0].read_text(encoding="utf-8").strip()
|
||||
if len(latest_output) > _MAX_CONTEXT_CHARS:
|
||||
latest_output = latest_output[:_MAX_CONTEXT_CHARS] + "\n\n[... output truncated ...]"
|
||||
latest_output = (
|
||||
latest_output[:_MAX_CONTEXT_CHARS] + "\n\n[... output truncated ...]")
|
||||
if not latest_output:
|
||||
continue # silent skip — empty output
|
||||
if is_self:
|
||||
prompt = _prepend_context_block(
|
||||
prompt, "Your previous run's output", _SELF_CONTEXT_INTRO, latest_output,
|
||||
)
|
||||
prompt, "Your previous run's output", _SELF_CONTEXT_INTRO, latest_output)
|
||||
else:
|
||||
prompt = _prepend_context_block(
|
||||
prompt, f"Output from job '{source_job_id}'", _UPSTREAM_CONTEXT_INTRO, latest_output,
|
||||
prompt, f"Output from job '{source_job_id}'", _UPSTREAM_CONTEXT_INTRO,
|
||||
latest_output,
|
||||
)
|
||||
injected = True
|
||||
except (OSError, PermissionError) as e:
|
||||
@@ -107,7 +110,7 @@ def _inject_context_from(job: dict, prompt: str) -> tuple[str, bool]:
|
||||
|
||||
|
||||
def _load_cron_skill_parts(job: dict, skill_names: list[str]) -> list[str]:
|
||||
"""Load each named skill/bundle into prompt parts; unknown ones are skipped with a user notice."""
|
||||
"""Load each named skill/bundle into prompt parts; unknown ones are skipped with a notice."""
|
||||
from tools.skills_tool import skill_view
|
||||
from tools.skill_usage import bump_use
|
||||
from agent.skill_bundles import build_bundle_invocation_message, resolve_bundle_command_key
|
||||
@@ -126,7 +129,8 @@ def _load_cron_skill_parts(job: dict, skill_names: list[str]) -> list[str]:
|
||||
# Bundles shadow same-slug skills, mirroring the CLI/gateway slash-command path.
|
||||
bundle_key = resolve_bundle_command_key(skill_name.lstrip("/"))
|
||||
if bundle_key:
|
||||
bundle_payload = build_bundle_invocation_message(bundle_key, user_instruction="", task_id=task_id)
|
||||
bundle_payload = build_bundle_invocation_message(
|
||||
bundle_key, user_instruction="", task_id=task_id)
|
||||
if bundle_payload:
|
||||
if parts:
|
||||
parts.append("")
|
||||
@@ -141,7 +145,9 @@ def _load_cron_skill_parts(job: dict, skill_names: list[str]) -> list[str]:
|
||||
_skip("skill '%s' returned invalid JSON, skipping", skill_name)
|
||||
continue
|
||||
if not loaded.get("success"):
|
||||
_skip("skill not found, skipping — %s", loaded.get("error") or f"Failed to load skill '{skill_name}'")
|
||||
_skip(
|
||||
"skill not found, skipping — %s",
|
||||
loaded.get("error") or f"Failed to load skill '{skill_name}'")
|
||||
continue
|
||||
|
||||
try:
|
||||
@@ -154,8 +160,7 @@ def _load_cron_skill_parts(job: dict, skill_names: list[str]) -> list[str]:
|
||||
parts.extend([
|
||||
f'[IMPORTANT: The user has invoked the "{skill_name}" skill, indicating they want you to follow its instructions. The full skill content is loaded below.]',
|
||||
"",
|
||||
str(loaded.get("content") or "").strip(),
|
||||
])
|
||||
str(loaded.get("content") or "").strip()])
|
||||
|
||||
if skipped:
|
||||
parts.insert(0, (
|
||||
@@ -181,16 +186,11 @@ _CRON_HINT = (
|
||||
|
||||
|
||||
def _build_job_prompt(
|
||||
job: dict,
|
||||
prerun_script: Optional[tuple] = None,
|
||||
extra_prompt: Optional[str] = None,
|
||||
) -> str:
|
||||
job: dict, prerun_script: Optional[tuple] = None, extra_prompt: Optional[str] = None) -> str:
|
||||
"""Build the effective prompt for a cron job, optionally loading skills first.
|
||||
|
||||
``prerun_script``: cached ``(success, stdout)`` from a script the caller already ran (wake-gate
|
||||
check) — skips re-execution. ``extra_prompt``: per-run ``## Run Context`` for this fire only,
|
||||
never persisted to the job.
|
||||
"""
|
||||
never persisted to the job."""
|
||||
user_prompt = str(job.get("prompt") or "")
|
||||
if extra_prompt:
|
||||
user_prompt = f"{user_prompt}\n\n## Run Context\n{extra_prompt}"
|
||||
@@ -202,12 +202,12 @@ def _build_job_prompt(
|
||||
script_path = job.get("script")
|
||||
if script_path:
|
||||
success, script_output = (
|
||||
prerun_script if prerun_script is not None else _sched._run_job_script(script_path)
|
||||
)
|
||||
prerun_script if prerun_script is not None else _sched._run_job_script(script_path))
|
||||
if success and not script_output:
|
||||
return None # no output → nothing to report, skip the AI call
|
||||
heading, intro = (
|
||||
("Script Output", "The following data was collected by a pre-run script. Use it as context for your analysis.")
|
||||
("Script Output", "The following data was collected by a pre-run script. "
|
||||
"Use it as context for your analysis.")
|
||||
if success
|
||||
else ("Script Error", "The data-collection script failed. Report this to the user.")
|
||||
)
|
||||
@@ -229,7 +229,8 @@ def _build_job_prompt(
|
||||
skill_names = _job_skill_names(job)
|
||||
if not skill_names:
|
||||
return _scan_assembled_cron_prompt(
|
||||
prompt, job, has_skills=False, has_injected_data=has_injected_data, user_prompt=user_prompt,
|
||||
prompt, job, has_skills=False, has_injected_data=has_injected_data,
|
||||
user_prompt=user_prompt,
|
||||
)
|
||||
|
||||
parts = _load_cron_skill_parts(job, skill_names)
|
||||
@@ -242,7 +243,11 @@ def _build_job_prompt(
|
||||
# Declare that boundary for the Anthropic cache planner.
|
||||
stable_prefix = append_user_instruction(parts, prompt)
|
||||
assembled = _scan_assembled_cron_prompt("\n".join(parts), job, has_skills=True)
|
||||
if stable_prefix and len(assembled) > len(stable_prefix) and assembled.startswith(stable_prefix):
|
||||
if (
|
||||
stable_prefix
|
||||
and len(assembled) > len(stable_prefix)
|
||||
and assembled.startswith(stable_prefix)
|
||||
):
|
||||
# Guarded: the scanner may mutate the bytes; mismatch → whole-message caching.
|
||||
from agent.prompt_cache_boundary import register_stable_prefix
|
||||
|
||||
@@ -251,21 +256,15 @@ def _build_job_prompt(
|
||||
|
||||
|
||||
def _scan_assembled_cron_prompt(
|
||||
assembled: str,
|
||||
job: dict,
|
||||
*,
|
||||
has_skills: bool = False,
|
||||
has_injected_data: bool = False,
|
||||
assembled: str, job: dict, *, has_skills: bool = False, has_injected_data: bool = False,
|
||||
user_prompt: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Scan the assembled cron prompt for injection; raise ``CronPromptInjectionBlocked`` on a hit.
|
||||
|
||||
Needed because skill content is loaded from disk at runtime (never scanned at create/update)
|
||||
and cron auto-approves tool calls. Tier is chosen by what the prompt CONTAINS: user prompt +
|
||||
hint only → STRICT ``_scan_cron_prompt``; skills or injected data → LOOSER
|
||||
``_scan_cron_skill_assembled`` (command-shape patterns dropped, invisible unicode sanitized not
|
||||
blocked, so a false positive cannot permanently kill a job). With injected data but no skills,
|
||||
``user_prompt`` is additionally scanned STRICT (defense-in-depth for legacy jobs).
|
||||
and cron auto-approves tool calls. Tier by what the prompt CONTAINS: user prompt + hint only →
|
||||
STRICT ``_scan_cron_prompt``; skills or injected data → LOOSER ``_scan_cron_skill_assembled``
|
||||
(command-shape patterns dropped, invisible unicode sanitized not blocked, so a false positive
|
||||
cannot permanently kill a job); injected data without skills also scans ``user_prompt`` STRICT.
|
||||
"""
|
||||
from tools.cronjob_tools import _scan_cron_prompt, _scan_cron_skill_assembled
|
||||
|
||||
@@ -279,8 +278,7 @@ def _scan_assembled_cron_prompt(
|
||||
if scan_error:
|
||||
logger.warning(
|
||||
"Cron job '%s': assembled prompt blocked by injection scanner — %s",
|
||||
job.get("name") or job.get("id") or "<unknown>", scan_error,
|
||||
)
|
||||
job.get("name") or job.get("id") or "<unknown>", scan_error)
|
||||
raise _sched.CronPromptInjectionBlocked(scan_error)
|
||||
return assembled
|
||||
|
||||
@@ -294,8 +292,8 @@ def _guard_job_credential_exfil(job: dict) -> None:
|
||||
from tools.cronjob_tools import _validate_cron_base_url
|
||||
err = _validate_cron_base_url(job.get("provider"), job.get("base_url"))
|
||||
except Exception as exc:
|
||||
# Fail CLOSED on validator/import errors — but only for jobs WITH a base_url override; a job
|
||||
# without one cannot exfiltrate via this path, so it still runs.
|
||||
# Fail CLOSED on validator/import errors — but only for jobs WITH a base_url override;
|
||||
# a job without one cannot exfiltrate via this path, so it still runs.
|
||||
err = (
|
||||
f"could not validate provider/base_url pair "
|
||||
f"({exc.__class__.__name__}: {exc}); refusing to run a job with "
|
||||
@@ -306,14 +304,12 @@ def _guard_job_credential_exfil(job: dict) -> None:
|
||||
logger.error(
|
||||
"Job '%s': refusing to run — unsafe provider/base_url pair could "
|
||||
"exfiltrate a stored credential: %s",
|
||||
job_id, err,
|
||||
)
|
||||
job_id, err)
|
||||
raise RuntimeError(f"Cron job '{job_id}' blocked for safety: {err}")
|
||||
|
||||
|
||||
def _block_and_pause_job(
|
||||
job_id: str, job_name: str, reason: str
|
||||
) -> tuple[bool, str, str, Optional[str]]:
|
||||
job_id: str, job_name: str, reason: str) -> tuple[bool, str, str, Optional[str]]:
|
||||
"""Fail a run closed and pause the job: an unrunnable job left enabled re-fires every tick
|
||||
forever; ``paused_at``/``paused_reason`` give an auditable record."""
|
||||
from cron.jobs import pause_job
|
||||
|
||||
+57
-62
@@ -37,9 +37,10 @@ def _positive_int(raw) -> Optional[int]:
|
||||
return timeout if timeout > 0 else None
|
||||
|
||||
|
||||
def _timeout_from_env_or_config(env_var: str, config_key: str, parse: Callable[[Any], Any], label: str):
|
||||
def _timeout_from_env_or_config(
|
||||
env_var: str, config_key: str, parse: Callable[[Any], Any], label: str):
|
||||
"""Shared env → ``cron.<config_key>`` resolution. ``parse`` returns the value or None to keep
|
||||
looking; a parse error on the env var WARNs, on config DEBUGs. Returns None when neither yields."""
|
||||
looking; a parse error on the env var WARNs, on config DEBUGs. None when neither yields."""
|
||||
env_value = os.getenv(env_var, "").strip()
|
||||
if env_value:
|
||||
try:
|
||||
@@ -69,9 +70,12 @@ def _get_script_timeout() -> int:
|
||||
if timeout is not None:
|
||||
return timeout
|
||||
except Exception:
|
||||
logger.warning("Invalid patched _SCRIPT_TIMEOUT=%r; using env/config/default", _sched._SCRIPT_TIMEOUT)
|
||||
logger.warning(
|
||||
"Invalid patched _SCRIPT_TIMEOUT=%r; using env/config/default",
|
||||
_sched._SCRIPT_TIMEOUT)
|
||||
resolved = _timeout_from_env_or_config(
|
||||
"HERMES_CRON_SCRIPT_TIMEOUT", "script_timeout_seconds", _positive_int, "cron script timeout",
|
||||
"HERMES_CRON_SCRIPT_TIMEOUT", "script_timeout_seconds", _positive_int,
|
||||
"cron script timeout",
|
||||
)
|
||||
return _sched._DEFAULT_SCRIPT_TIMEOUT if resolved is None else resolved
|
||||
|
||||
@@ -84,8 +88,7 @@ def _get_media_send_timeout() -> int:
|
||||
``cron.media_send_timeout_seconds``, then 300s (long TTS audio can exceed a 30s window)."""
|
||||
resolved = _timeout_from_env_or_config(
|
||||
"HERMES_CRON_MEDIA_SEND_TIMEOUT", "media_send_timeout_seconds", _positive_int,
|
||||
"cron media-send timeout",
|
||||
)
|
||||
"cron media-send timeout")
|
||||
return _DEFAULT_MEDIA_SEND_TIMEOUT if resolved is None else resolved
|
||||
|
||||
|
||||
@@ -95,8 +98,7 @@ def _get_session_db_timeout() -> float:
|
||||
0 is meaningful (unlimited, debugging opt-in), so values pass through untouched."""
|
||||
resolved = _timeout_from_env_or_config(
|
||||
"HERMES_CRON_SESSION_DB_TIMEOUT", "session_db_timeout_seconds", float,
|
||||
"cron.session_db_timeout_seconds",
|
||||
)
|
||||
"cron.session_db_timeout_seconds")
|
||||
return 10.0 if resolved is None else resolved
|
||||
|
||||
|
||||
@@ -113,8 +115,8 @@ def _read_windows_pyvenv_cfg(venv_dir: Path) -> dict[str, str]:
|
||||
|
||||
def _windows_cron_python_invocation(python_exe: str) -> tuple[str, dict[str, str]]:
|
||||
"""Hidden, output-capable Python invocation for Windows cron scripts. ``pythonw.exe`` loses
|
||||
captured output; uv venv launchers can re-exec the base console python and flash a window even
|
||||
with CREATE_NO_WINDOW, so run the base python directly with venv paths overlaid in env."""
|
||||
captured output; uv venv launchers can re-exec the base console python and flash a window
|
||||
even with CREATE_NO_WINDOW, so run the base python directly with venv paths overlaid in env."""
|
||||
if sys.platform != "win32":
|
||||
return python_exe, {}
|
||||
|
||||
@@ -135,7 +137,8 @@ def _windows_cron_python_invocation(python_exe: str) -> tuple[str, dict[str, str
|
||||
if base_python.exists() and site_packages.exists():
|
||||
interpreter = base_python
|
||||
env_overlay["VIRTUAL_ENV"] = str(venv_dir)
|
||||
pythonpath_entries = [str(_sched.Path(__file__).resolve().parents[1]), str(site_packages)]
|
||||
pythonpath_entries = [
|
||||
str(_sched.Path(__file__).resolve().parents[1]), str(site_packages)]
|
||||
existing_pythonpath = os.environ.get("PYTHONPATH", "")
|
||||
if existing_pythonpath:
|
||||
pythonpath_entries.append(existing_pythonpath)
|
||||
@@ -148,16 +151,15 @@ def _terminate_cron_script_process(proc: subprocess.Popen) -> None:
|
||||
"""Best-effort hard stop of a cron script and every child it spawned."""
|
||||
if proc.poll() is not None:
|
||||
return
|
||||
if sys.platform == "win32":
|
||||
if sys.platform != "win32":
|
||||
_terminate_process_group(proc)
|
||||
else:
|
||||
try:
|
||||
subprocess.run(
|
||||
["taskkill", "/PID", str(proc.pid), "/T", "/F"],
|
||||
capture_output=True, timeout=10, creationflags=_sched.windows_hide_flags(), check=False,
|
||||
)
|
||||
["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=10,
|
||||
creationflags=_sched.windows_hide_flags(), check=False)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
proc.kill()
|
||||
else:
|
||||
_terminate_process_group(proc)
|
||||
try:
|
||||
proc.wait(timeout=1.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
@@ -170,7 +172,7 @@ def _terminate_process_group(proc: subprocess.Popen) -> None:
|
||||
the pipe write ends open and the caller's communicate() would block on EOF forever)."""
|
||||
try:
|
||||
process_group = os.getpgid(proc.pid)
|
||||
os.killpg(process_group, signal.SIGTERM) # windows-footgun: ok — POSIX-only branch (win32 handled by caller)
|
||||
os.killpg(process_group, signal.SIGTERM) # windows-footgun: ok — POSIX-only branch
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
return
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
@@ -191,9 +193,9 @@ def _terminate_cron_script_tree(proc: subprocess.Popen) -> None:
|
||||
pid = getattr(proc, "pid", None)
|
||||
if not isinstance(pid, int) or pid <= 0:
|
||||
logger.warning(
|
||||
"Cron script tree-kill received invalid pid %r; falling back to process-group termination",
|
||||
pid,
|
||||
)
|
||||
"Cron script tree-kill received invalid pid %r; "
|
||||
"falling back to process-group termination",
|
||||
pid)
|
||||
_sched._terminate_cron_script_process(proc)
|
||||
return
|
||||
try:
|
||||
@@ -202,23 +204,22 @@ def _terminate_cron_script_tree(proc: subprocess.Popen) -> None:
|
||||
from agent.deadline import kill_process_tree
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"agent.deadline.kill_process_tree unavailable; falling back to process-group termination",
|
||||
exc_info=True,
|
||||
)
|
||||
"agent.deadline.kill_process_tree unavailable; "
|
||||
"falling back to process-group termination",
|
||||
exc_info=True)
|
||||
_sched._terminate_cron_script_process(proc)
|
||||
return
|
||||
try:
|
||||
if kill_process_tree(pid):
|
||||
return
|
||||
logger.warning(
|
||||
"Cron script tree-kill reported no signal for pid %s; falling back to process-group termination",
|
||||
pid,
|
||||
)
|
||||
"Cron script tree-kill reported no signal for pid %s; "
|
||||
"falling back to process-group termination",
|
||||
pid)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Cron script tree-kill failed for pid %s; falling back to process-group termination",
|
||||
pid, exc_info=True,
|
||||
)
|
||||
pid, exc_info=True)
|
||||
_sched._terminate_cron_script_process(proc)
|
||||
|
||||
|
||||
@@ -238,20 +239,19 @@ def _drain_script_pipes(proc: subprocess.Popen) -> None:
|
||||
proc.wait(timeout=5.0)
|
||||
|
||||
|
||||
def _windows_cron_bootstrap_argv(python_exe: str, env_overlay: dict[str, str], script_path: str) -> list[str]:
|
||||
"""Bootstrap a cron script under the base interpreter with ``.pth`` support.
|
||||
Overlay mode runs base ``python.exe`` (avoids the launcher flashing a console window) with the
|
||||
venv on ``PYTHONPATH`` — but ``.pth`` files are only processed by ``site.addsitedir()``, so
|
||||
editable installs would be invisible. Bootstrap via addsitedir + ``runpy.run_path`` (keeps
|
||||
``__file__`` and ``sys.path[0]`` semantics); plain invocation if the venv is unresolvable."""
|
||||
def _windows_cron_bootstrap_argv(
|
||||
python_exe: str, env_overlay: dict[str, str], script_path: str) -> list[str]:
|
||||
"""Bootstrap a cron script under the base interpreter with ``.pth`` support. Overlay mode puts
|
||||
the venv on ``PYTHONPATH``, but ``.pth`` files are only processed by ``site.addsitedir()``, so
|
||||
editable installs would be invisible; bootstrap via addsitedir + ``runpy.run_path`` (keeps
|
||||
``__file__``/``sys.path[0]`` semantics). Plain invocation if the venv is unresolvable."""
|
||||
site_packages = _sched.Path(env_overlay.get("VIRTUAL_ENV", "")) / "Lib" / "site-packages"
|
||||
if not site_packages.is_dir():
|
||||
# Warn: silent fallback would make "editable installs invisible" undiagnosable.
|
||||
logger.warning(
|
||||
"Windows cron script: venv site-packages %s not found; running "
|
||||
"without .pth processing (editable installs may be unimportable)",
|
||||
site_packages,
|
||||
)
|
||||
site_packages)
|
||||
return [python_exe, script_path]
|
||||
bootstrap = (
|
||||
"import os, runpy, site, sys;"
|
||||
@@ -265,9 +265,9 @@ def _windows_cron_bootstrap_argv(python_exe: str, env_overlay: dict[str, str], s
|
||||
|
||||
|
||||
def _resolve_script_path(script_path: str) -> tuple[Optional[Path], Optional[str]]:
|
||||
"""Validate a job script path; ``(path, None)`` or ``(None, error)``. Scripts MUST resolve inside
|
||||
HERMES_HOME/scripts/ (relative, absolute and ``~`` paths are all validated — path traversal /
|
||||
absolute-path injection). Same contract as cron.lifecycle_guard._expand_candidate_path."""
|
||||
"""Validate a job script path; ``(path, None)`` or ``(None, error)``. Scripts MUST resolve
|
||||
inside HERMES_HOME/scripts/ (relative, absolute and ``~`` paths are all validated — path
|
||||
traversal / absolute-path injection); contract of lifecycle_guard._expand_candidate_path."""
|
||||
scripts_dir = _sched._get_hermes_home() / "scripts"
|
||||
_ensure_cron_dir(scripts_dir)
|
||||
scripts_dir_resolved = scripts_dir.resolve()
|
||||
@@ -299,9 +299,9 @@ def _resolve_script_path(script_path: str) -> tuple[Optional[Path], Optional[str
|
||||
|
||||
|
||||
def _script_argv(path: Path) -> tuple[Optional[list[str]], dict[str, str], Optional[str]]:
|
||||
"""``(argv, env_overlay, error)`` for a validated script. Interpreter by extension — the shebang
|
||||
is deliberately NOT honoured (small, auditable surface): ``.sh``/``.bash`` → bash, else
|
||||
``sys.executable`` (Windows uv-venv overlay gets the .pth bootstrap)."""
|
||||
"""``(argv, env_overlay, error)`` for a validated script. Interpreter by extension — the
|
||||
shebang is deliberately NOT honoured (small, auditable surface): ``.sh``/``.bash`` → bash,
|
||||
else ``sys.executable`` (Windows uv-venv overlay gets the .pth bootstrap)."""
|
||||
if path.suffix.lower() in {".sh", ".bash"}:
|
||||
# which() finds Git Bash on Windows; None there → clear error instead of a "[WinError 2]".
|
||||
_bash = shutil.which("bash") or ("/bin/bash" if os.path.isfile("/bin/bash") else None)
|
||||
@@ -319,8 +319,7 @@ def _script_argv(path: Path) -> tuple[Optional[list[str]], dict[str, str], Optio
|
||||
|
||||
|
||||
def _run_job_script(
|
||||
script_path: str,
|
||||
workdir: Optional[str] = None,
|
||||
script_path: str, workdir: Optional[str] = None,
|
||||
cancel_event: Optional[_CancelEventLike] = None,
|
||||
) -> tuple[bool, str]:
|
||||
"""Execute a cron job's script and return ``(success, output)``; on failure *output* is the
|
||||
@@ -341,17 +340,16 @@ def _run_job_script(
|
||||
popen_kwargs: dict[str, Any] = {"start_new_session": True}
|
||||
if sys.platform == "win32":
|
||||
popen_kwargs = {
|
||||
"creationflags": _sched.windows_hide_flags() | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0),
|
||||
"creationflags": _sched.windows_hide_flags()
|
||||
| getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0),
|
||||
"encoding": "utf-8",
|
||||
"errors": "replace",
|
||||
}
|
||||
"errors": "replace"}
|
||||
env = build_subprocess_env()
|
||||
env.update(env_overlay)
|
||||
# Subprocess cwd only (default: scripts-dir parent). NEVER os.chdir() the process.
|
||||
proc = subprocess.Popen(
|
||||
argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
cwd=workdir or str(path.parent), env=env, **popen_kwargs,
|
||||
)
|
||||
cwd=workdir or str(path.parent), env=env, **popen_kwargs)
|
||||
deadline = time.monotonic() + script_timeout
|
||||
while True:
|
||||
# Tree-kill on cancel AND timeout: killpg misses setsid grandchildren (watchdogs,
|
||||
@@ -399,7 +397,8 @@ def _start_heartbeat_thread(loop_fn, name: str, fail_log) -> Optional[threading.
|
||||
"""Start ``loop_fn`` on a daemon thread inside a copy of the current context (multiplexed
|
||||
profile ContextVars). On failure calls ``fail_log()`` inside the except (traceback intact) and
|
||||
returns None."""
|
||||
thread = threading.Thread(target=contextvars.copy_context().run, args=(loop_fn,), name=name, daemon=True)
|
||||
thread = threading.Thread(
|
||||
target=contextvars.copy_context().run, args=(loop_fn,), name=name, daemon=True)
|
||||
try:
|
||||
thread.start()
|
||||
except Exception:
|
||||
@@ -409,18 +408,13 @@ def _start_heartbeat_thread(loop_fn, name: str, fail_log) -> Optional[threading.
|
||||
|
||||
|
||||
def _run_job_script_with_claim_heartbeat(
|
||||
job: dict,
|
||||
script_path: str,
|
||||
workdir: Optional[str] = None,
|
||||
job: dict, script_path: str, workdir: Optional[str] = None,
|
||||
cancel_event: Optional[_CancelEventLike] = None,
|
||||
) -> tuple[bool, str]:
|
||||
"""Run a cron script while heartbeating its owned one-shot claim.
|
||||
|
||||
A long script can outlive the stale-claim TTL; without a heartbeat another scheduler would
|
||||
re-dispatch the one-shot. Recurring/unclaimed runs have no durable claim → no thread. The owner
|
||||
is captured from the dispatched job, never re-read, so a stale runner cannot extend a
|
||||
replacement owner's claim.
|
||||
"""
|
||||
"""Run a cron script while heartbeating its owned one-shot claim. A long script can outlive
|
||||
the stale-claim TTL; without a heartbeat another scheduler would re-dispatch the one-shot.
|
||||
Recurring/unclaimed runs have no durable claim → no thread. The owner is captured from the
|
||||
dispatched job, never re-read, so a stale runner cannot extend a replacement owner's claim."""
|
||||
schedule = job.get("schedule")
|
||||
claim = job.get("run_claim")
|
||||
owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
|
||||
@@ -439,7 +433,8 @@ def _run_job_script_with_claim_heartbeat(
|
||||
|
||||
heartbeat_thread = _start_heartbeat_thread(
|
||||
_heartbeat_loop, "cron-script-claim-heartbeat",
|
||||
lambda: logger.debug("Job '%s': could not start script run_claim heartbeat", job_id, exc_info=True),
|
||||
lambda: logger.debug(
|
||||
"Job '%s': could not start script run_claim heartbeat", job_id, exc_info=True),
|
||||
)
|
||||
if heartbeat_thread is None:
|
||||
return _sched._run_job_script(script_path, workdir=workdir, cancel_event=cancel_event)
|
||||
|
||||
Reference in New Issue
Block a user