fix(packaging): derive root py-modules from the tree in setup.py
The static py-modules list in pyproject.toml drifted from the source tree each time the root layout changed. An installed wheel then raised ModuleNotFoundError on import: hermes_state_common, and every gateway or CLI start failed. The list missed hermes_state_holders, mini_swe_runner, and the 15 new hermes_state_* modules from this branch. setup.py now derives py_modules from the source tree at build time. setuptools package discovery sees only directories with an __init__.py, so root single-file modules need py_modules in every wheel build. setup() kwargs merge with pyproject.toml, and setup.py is the only legitimate wheel or sdist builder, so the derived list is the single source of truth. The nix build is the only wheel consumer. Its source filter keeps every root .py file, so the build sandbox derives the same set as the checkout. Editable installs do not read py_modules: build_editable never runs bdist_wheel. Verified: wheel and sdist built with HERMES_NIX_BUILD=1 carry all 38 root modules. The guard still blocks builds without the nix env var. The sealed uv2nix venv from nix build .#default imports hermes_state_holders, hermes_state_sessions, mini_swe_runner, and hermes_startup_watchdog. (cherry picked from commit e23d467b523d50ad088b89b578017996709589cf) Invariant test updated to pin the derived list (no static py-modules; every root module packaged code imports ships).
This commit is contained in:
ethernet
+3
-3
@@ -9,9 +9,9 @@ Usage:
|
||||
"""
|
||||
|
||||
# hermes_bootstrap must be the very first import — it sets up UTF-8 stdio on
|
||||
# Windows (no-op on POSIX). Guarded: it is a pyproject ``py-modules`` entry, so
|
||||
# after a ``git pull`` / interrupted ``hermes update`` the editable install's
|
||||
# ``.pth`` may not list it yet; crashing here would block ``hermes update``.
|
||||
# Windows (no-op on POSIX). Guarded: after a ``git pull`` / interrupted
|
||||
# ``hermes update`` the editable install's ``.pth`` may not list it yet; crashing
|
||||
# here would block ``hermes update``.
|
||||
try:
|
||||
import hermes_bootstrap # noqa: F401
|
||||
except ModuleNotFoundError:
|
||||
|
||||
+4
-41
@@ -558,47 +558,10 @@ youtube-transcript-api = false
|
||||
|
||||
|
||||
[tool.setuptools]
|
||||
# Top-level single-file modules (not packages). Without this, uv2nix's
|
||||
# sealed venv is missing hermes_constants, run_agent, etc.
|
||||
py-modules = [
|
||||
"run_agent",
|
||||
"registration_lifecycle",
|
||||
"model_tools",
|
||||
"toolsets",
|
||||
"batch_runner",
|
||||
"trajectory_compressor",
|
||||
"toolset_distributions",
|
||||
"cli",
|
||||
"hermes_bootstrap",
|
||||
"hermes_constants",
|
||||
"hermes_state",
|
||||
"hermes_state_common",
|
||||
"hermes_state_compression",
|
||||
"hermes_state_dbfile",
|
||||
"hermes_state_errors",
|
||||
"hermes_state_fts",
|
||||
"hermes_state_gateway",
|
||||
"hermes_state_guard",
|
||||
"hermes_state_holders",
|
||||
"hermes_state_maintenance",
|
||||
"hermes_state_messages",
|
||||
"hermes_state_portability",
|
||||
"hermes_state_readpool",
|
||||
"hermes_state_registry",
|
||||
"hermes_state_repair",
|
||||
"hermes_state_schema",
|
||||
"hermes_state_search",
|
||||
"hermes_state_sessions",
|
||||
"hermes_state_telegram",
|
||||
"hermes_state_titles",
|
||||
"hermes_state_usage",
|
||||
"hermes_state_wal",
|
||||
"hermes_startup_watchdog",
|
||||
"hermes_time",
|
||||
"hermes_logging",
|
||||
"utils",
|
||||
"mcp_serve",
|
||||
]
|
||||
# Root single-file modules are derived by setup.py at build time from the
|
||||
# source tree (see the ``_root_py_modules`` helper there). A static
|
||||
# ``py-modules`` list here drifted from the tree each time the layout changed,
|
||||
# and the drift broke installed wheels. Do not add the list back.
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["agent", "agent.*", "tools", "tools.*", "hermes_cli", "hermes_cli.*", "gateway", "gateway.*", "tui_gateway", "tui_gateway.*", "cron", "cron.*", "acp_adapter", "plugins", "plugins.*", "providers", "providers.*"]
|
||||
|
||||
@@ -29,6 +29,8 @@ import os
|
||||
from setuptools import setup
|
||||
from setuptools.command.sdist import sdist
|
||||
|
||||
_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
_IN_NIX_BUILD = os.environ.get("HERMES_NIX_BUILD") == "1"
|
||||
|
||||
_BLOCK_MESSAGE = (
|
||||
@@ -71,4 +73,27 @@ try:
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
setup(cmdclass=cmdclass)
|
||||
# Root single-file modules (``run_agent``, ``hermes_state``, ``toolsets``...)
|
||||
# are invisible to ``packages.find``: that finder sees only directories with an
|
||||
# ``__init__.py``. The wheel build needs them on ``py_modules``, so derive the
|
||||
# list from the source tree at build time. A static list in ``pyproject.toml``
|
||||
# drifted each time the tree layout changed (missing modules broke installed
|
||||
# wheels with ``ModuleNotFoundError``), so there is no list to maintain here.
|
||||
# ``setup()`` kwargs merge with ``pyproject.toml``, and this file is the only
|
||||
# legitimate wheel/sdist builder, so the derived value is the single source.
|
||||
# Editable installs do not read it: ``build_editable`` never runs
|
||||
# ``bdist_wheel``. The filter source (``nix/lib.nix`` ``pythonSrc``) keeps
|
||||
# every root ``.py`` file, so the build sandbox sees the same set of files.
|
||||
def _root_py_modules():
|
||||
try:
|
||||
names = os.listdir(_ROOT)
|
||||
except OSError:
|
||||
return []
|
||||
return sorted(
|
||||
name[:-3]
|
||||
for name in names
|
||||
if name.endswith(".py") and name != "setup.py"
|
||||
)
|
||||
|
||||
|
||||
setup(cmdclass=cmdclass, py_modules=_root_py_modules())
|
||||
|
||||
@@ -1,95 +1,74 @@
|
||||
"""Invariant: every root-level module the packaged code imports ships in the wheel.
|
||||
"""Packaging invariant: every root-level module that packaged code imports ships in the wheel.
|
||||
|
||||
The repo keeps a handful of single-file modules at the repository root
|
||||
(``hermes_state.py``, ``run_agent.py``, ``cli.py``, ...). setuptools'
|
||||
``packages.find`` never picks those up, so ``[tool.setuptools] py-modules``
|
||||
in pyproject.toml has to name each one explicitly. When a root module is
|
||||
split into siblings (``hermes_state_*``) and the list is not updated, the
|
||||
source tree still works (cwd is on ``sys.path``) but the built wheel/sdist
|
||||
and the uv2nix sealed venv raise ``ModuleNotFoundError`` on the very first
|
||||
``import hermes_state``.
|
||||
|
||||
This test derives the required set from the code rather than freezing a
|
||||
list: parse pyproject, walk every import (top-level *and* lazy/function-
|
||||
scoped) in the packaged root modules and packaged packages, and require
|
||||
that any import resolving to a root-level ``*.py`` file is declared.
|
||||
``packages.find`` only sees directories with ``__init__.py``; root single-file modules
|
||||
(``run_agent``, ``hermes_state``, ``toolsets``...) reach the wheel through
|
||||
``setup.py::_root_py_modules()``, which derives the list from the tree at build time. A
|
||||
static list drifted every time the root layout changed and broke installed wheels with
|
||||
``ModuleNotFoundError`` on ``import hermes_state``. This test pins the two halves of that
|
||||
contract: the derived list covers every root module packaged code can import, and the
|
||||
helper stays the single source (no static ``py-modules`` creeping back into pyproject).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import importlib.util
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
PACKAGES = ("agent", "tools", "hermes_cli", "gateway", "tui_gateway", "cron", "acp_adapter", "plugins", "providers")
|
||||
|
||||
|
||||
def _pyproject() -> dict:
|
||||
return tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))
|
||||
def _root_py_modules() -> set[str]:
|
||||
spec = importlib.util.spec_from_file_location("_hermes_setup_py", REPO_ROOT / "setup.py")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
saved = sys.argv
|
||||
sys.argv = ["setup.py", "--name"] # setup() must not try to build anything on import
|
||||
try:
|
||||
try:
|
||||
spec.loader.exec_module(mod)
|
||||
except SystemExit:
|
||||
pass
|
||||
finally:
|
||||
sys.argv = saved
|
||||
return set(mod._root_py_modules())
|
||||
|
||||
|
||||
def _declared_py_modules() -> set[str]:
|
||||
return set(_pyproject()["tool"]["setuptools"]["py-modules"])
|
||||
|
||||
|
||||
def _packaged_package_roots() -> list[Path]:
|
||||
include = _pyproject()["tool"]["setuptools"]["packages"]["find"]["include"]
|
||||
roots = []
|
||||
for pattern in include:
|
||||
top = pattern.split(".", 1)[0]
|
||||
if "*" in top:
|
||||
def _imported_root_names(paths) -> dict[str, set[str]]:
|
||||
root_files = {p.stem for p in REPO_ROOT.glob("*.py")}
|
||||
hits: dict[str, set[str]] = {}
|
||||
for path in paths:
|
||||
try:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))
|
||||
except SyntaxError:
|
||||
continue
|
||||
path = REPO_ROOT / top
|
||||
if path.is_dir() and path not in roots:
|
||||
roots.append(path)
|
||||
return roots
|
||||
|
||||
|
||||
def _root_module_names() -> set[str]:
|
||||
return {p.stem for p in REPO_ROOT.glob("*.py")}
|
||||
|
||||
|
||||
def _imported_top_names(path: Path) -> set[str]:
|
||||
"""All top-level names imported anywhere in ``path`` (absolute imports only)."""
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
names: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
names = []
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
names.add(alias.name.split(".", 1)[0])
|
||||
names = [a.name.split(".")[0] for a in node.names]
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
||||
names.add(node.module.split(".", 1)[0])
|
||||
return names
|
||||
names = [node.module.split(".")[0]]
|
||||
for n in names:
|
||||
if n in root_files:
|
||||
hits.setdefault(str(path.relative_to(REPO_ROOT)), set()).add(n)
|
||||
return hits
|
||||
|
||||
|
||||
def _packaged_source_files() -> list[Path]:
|
||||
files = [REPO_ROOT / f"{name}.py" for name in _declared_py_modules()]
|
||||
for root in _packaged_package_roots():
|
||||
files.extend(root.rglob("*.py"))
|
||||
return [f for f in files if f.is_file()]
|
||||
|
||||
|
||||
def test_declared_py_modules_exist_at_repo_root():
|
||||
missing = sorted(n for n in _declared_py_modules() if not (REPO_ROOT / f"{n}.py").is_file())
|
||||
assert not missing, f"py-modules names files that do not exist at the repo root: {missing}"
|
||||
|
||||
|
||||
def test_every_root_module_imported_by_packaged_code_is_in_py_modules():
|
||||
declared = _declared_py_modules()
|
||||
root_modules = _root_module_names()
|
||||
|
||||
# importer -> set of undeclared root modules it imports
|
||||
offenders: dict[str, set[str]] = {}
|
||||
for src in _packaged_source_files():
|
||||
needed = _imported_top_names(src) & root_modules
|
||||
undeclared = needed - declared
|
||||
if undeclared:
|
||||
offenders[str(src.relative_to(REPO_ROOT))] = undeclared
|
||||
|
||||
assert not offenders, (
|
||||
"Root-level modules are imported by packaged code but missing from "
|
||||
"[tool.setuptools] py-modules in pyproject.toml. The source tree hides "
|
||||
"this (cwd is on sys.path); the built wheel / uv2nix sealed venv will "
|
||||
"fail with ModuleNotFoundError. Add them to py-modules:\n"
|
||||
+ "\n".join(f" {importer}: {sorted(mods)}" for importer, mods in sorted(offenders.items()))
|
||||
def test_pyproject_has_no_static_py_modules_list():
|
||||
cfg = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))
|
||||
assert "py-modules" not in cfg["tool"]["setuptools"], (
|
||||
"root modules are derived in setup.py::_root_py_modules(); a static py-modules list drifts "
|
||||
"from the tree and breaks installed wheels. Do not add it back."
|
||||
)
|
||||
|
||||
|
||||
def test_every_root_module_imported_by_packaged_code_is_shipped():
|
||||
shipped = _root_py_modules()
|
||||
paths = [REPO_ROOT / f"{n}.py" for n in shipped]
|
||||
for pkg in PACKAGES:
|
||||
paths.extend((REPO_ROOT / pkg).rglob("*.py"))
|
||||
missing = {f: sorted(n for n in names if n not in shipped) for f, names in _imported_root_names(paths).items()}
|
||||
missing = {f: v for f, v in missing.items() if v}
|
||||
assert not missing, f"packaged code imports root modules the wheel would not ship: {missing}"
|
||||
assert "hermes_state" in shipped and "setup" not in shipped
|
||||
|
||||
Reference in New Issue
Block a user