refactor(tools): group H pass 4 — shared _joined tail for stdlib extractors, v3 mime fold, blank squeeze

This commit is contained in:
Teknium
2026-09-02 22:59:19 -07:00
parent 37232a4853
commit 50357b3c7b
3 changed files with 17 additions and 31 deletions
+4 -6
View File
@@ -39,12 +39,10 @@ def main():
ref_audio = Path(args.ref_audio).expanduser()
ref_text_path = Path(args.ref_text).expanduser()
if not ref_audio.exists():
print(f"Error: reference audio not found: {ref_audio}", file=sys.stderr)
sys.exit(1)
if not ref_text_path.exists():
print(f"Error: reference text not found: {ref_text_path}", file=sys.stderr)
sys.exit(1)
for label, p in (("audio", ref_audio), ("text", ref_text_path)):
if not p.exists():
print(f"Error: reference {label} not found: {p}", file=sys.stderr)
sys.exit(1)
ref_text = ref_text_path.read_text(encoding="utf-8").strip()
try:
-5
View File
@@ -58,23 +58,19 @@ def check_package_for_malware(command: str, args: list) -> Optional[str]:
ecosystem = _infer_ecosystem(command)
if not ecosystem:
return None # not npx/uvx — skip
package, version = _parse_package_from_args(args, ecosystem)
if not package:
return None
cache_key = (ecosystem, package, version)
hit, cached = _cache_get(cache_key)
if hit:
return cached
try:
malware = _query_osv(package, ecosystem, version)
except Exception as exc:
# Fail-open; deliberately NOT cached — see _CACHE_TTL_S comment.
logger.debug("OSV check failed for %s/%s (allowing): %s", ecosystem, package, exc)
return None
result = None
if malware:
ids = ", ".join(m["id"] for m in malware[:3])
@@ -115,7 +111,6 @@ def _parse_package_from_args(args: list, ecosystem: str) -> Tuple[Optional[str],
continue
package_token = arg
break
if not package_token:
return None, None
parser = _PACKAGE_PARSERS.get(ecosystem)
+13 -20
View File
@@ -79,7 +79,6 @@ def _anydoc() -> Optional[Any]:
return None
try:
from tools.lazy_deps import ensure as _lazy_ensure
# prompt=False: read_file must never block on an install prompt.
_lazy_ensure("tool.doc_extract", prompt=False)
_anydoc_module = importlib.import_module("anydoc")
@@ -345,6 +344,13 @@ def _pdf_coverage_note_from_bytes(data: bytes, display_path: str) -> str:
return ""
def _joined(lines: list[str], empty_error: str) -> str:
"""Join extracted lines with a single trailing newline; raise when nothing non-blank."""
if not any(line.strip() for line in lines):
raise ExtractionError(empty_error)
return "\n".join(lines).rstrip("\n") + "\n"
def _source_text(source) -> str:
if isinstance(source, str):
return source
@@ -402,13 +408,9 @@ def _notebook_output_text(output: Any) -> str:
return ""
data = output.get("data")
if not isinstance(data, dict):
data = {}
if isinstance(output.get("text"), (str, list)):
data["text/plain"] = output["text"]
for v3_key, mime in _V3_MIME_KEYS:
if v3_key in output:
data[mime] = output[v3_key]
if not isinstance(data, dict): # legacy v3: mime payloads sit flat on the output dict
data = {"text/plain": output["text"]} if isinstance(output.get("text"), (str, list)) else {}
data.update((mime, output[k]) for k, mime in _V3_MIME_KEYS if k in output)
if "application/vnd.jupyter.widget-view+json" in data:
return "[interactive widget — omitted]"
# Prefer readable text: models consume text/plain far better than markup.
@@ -454,7 +456,6 @@ def _extract_notebook(path: str) -> str:
raise ExtractionError(f"Not a valid notebook: {exc}") from exc
if not isinstance(nb, dict):
raise ExtractionError("Notebook root is not an object")
raw_cells = nb.get("cells")
if isinstance(raw_cells, list):
cells = [(f".cells[{i}].outputs", cell) for i, cell in enumerate(raw_cells)]
@@ -465,7 +466,6 @@ def _extract_notebook(path: str) -> str:
for ci, cell in enumerate(ws.get("cells", []))]
if not cells:
raise ExtractionError("Notebook contains no cells")
nb_name = os.path.basename(path)
counts = dict.fromkeys(_CELL_LABELS, 0)
out: list[str] = []
@@ -482,9 +482,7 @@ def _extract_notebook(path: str) -> str:
rendered = _notebook_outputs(cell, jq_pointer, nb_name)
if rendered:
out.extend((f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""))
if not out:
raise ExtractionError("Notebook contains no readable cells")
return "\n".join(out).rstrip("\n") + "\n"
return _joined(out, "Notebook contains no readable cells")
@contextlib.contextmanager
@@ -523,9 +521,7 @@ def _extract_docx(path: str) -> str:
buf = [(node.text or "") if node.tag == f"{w}t" else breaks.get(node.tag, "")
for node in para.iter()]
lines.extend("".join(buf).split("\n"))
if not any(line.strip() for line in lines):
raise ExtractionError("DOCX contains no extractable text")
return "\n".join(lines).rstrip("\n") + "\n"
return _joined(lines, "DOCX contains no extractable text")
def _extract_xlsx(path: str) -> str:
@@ -556,10 +552,7 @@ def _extract_xlsx(path: str) -> str:
if not rows:
out.append("(empty)")
out.append("")
if not out:
raise ExtractionError("XLSX has no visible sheets with content")
return "\n".join(out).rstrip("\n") + "\n"
return _joined(out, "XLSX has no visible sheets with content")
def _col_index(ref: str) -> int: