fix(redact): mask secrets in Python mapping reprs

This commit is contained in:
Demerzel (backend-eng)
2026-07-25 13:58:05 +01:00
committed by Teknium
parent ee51e8bf8b
commit 47668280e7
2 changed files with 258 additions and 1 deletions
+127 -1
View File
@@ -367,6 +367,58 @@ def _should_redact_assignment(key: str, value: str, *, check_keyword: bool) -> b
_JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)"
_JSON_FIELD_RE = re.compile(rf'("{_JSON_KEY_NAMES}")\s*:\s*"([^"]+)"', re.IGNORECASE)
# Python ``repr`` uses single-quoted mapping fields, so opaque credentials in
# tracebacks and pytest failure introspection bypass the double-quoted JSON rule
# above: ``{'BRAVE_API_KEY': 'opaque-value'}``. Capture identifier-shaped keys
# here, then apply the canonical high-confidence key policy in the callback.
_PYTHON_REPR_SECRET_KEYS = frozenset({
"access_token",
"refresh_token",
"id_token",
"auth_token",
"token",
"api_key",
"apikey",
"client_secret",
"secret",
"password",
"passwd",
"private_key",
"credential",
"credentials",
"authorization",
"bearer",
"secret_value",
"raw_secret",
"secret_input",
"key_material",
})
_PYTHON_REPR_ENV_SUFFIXES = (
"_API_KEY",
"_TOKEN",
"_SECRET",
"_PASSWORD",
"_PASSWD",
"_CREDENTIAL",
"_CREDENTIALS",
)
_PYTHON_REPR_FIELD_RE = re.compile(
r"'(?P<key>[A-Za-z_][A-Za-z0-9_]*)'(?P<sep>\s*:\s*)"
r"(?:"
r"(?P<single_prefix>[bB]?)'(?P<single_value>(?:\\.|[^'\\])+)'"
r"|(?P<double_prefix>[bB]?)\"(?P<double_value>(?:\\.|[^\"\\])+)\""
r")"
)
# Terminal/process output normally uses ``code_file=True`` to preserve source.
# Add repr masking only to high-confidence diagnostic lines: pytest assertion
# introspection (``E ...``) and final Python exception lines.
_PYTEST_DIAGNOSTIC_LINE_RE = re.compile(r"^(?P<prefix>[ \t]*E[ \t]{2,})(?P<body>.*)$")
_PYTHON_EXCEPTION_LINE_RE = re.compile(
r"^(?P<prefix>(?:[A-Za-z_]\w*\.)*[A-Za-z_]\w*"
r"(?:Error|Exception|Warning):[ \t]*)(?P<body>.*)$"
)
# Authorization / Proxy-Authorization, any scheme or bare credential; header
# name and scheme word preserved. The credential class excludes quotes: pulling
# a closing quote into the mask turns value corruption into SYNTAX corruption
@@ -515,6 +567,65 @@ def _mask_token(token: str) -> str:
return mask_secret(token, head=6, tail=4, floor=18)
def _is_python_repr_secret_key(key: str) -> bool:
"""Return True for exact secret keys or uppercase env credential suffixes."""
if key.casefold() in _PYTHON_REPR_SECRET_KEYS:
return True
return key.isupper() and key.endswith(_PYTHON_REPR_ENV_SUFFIXES)
def _redact_python_repr_fields(text: str) -> str:
"""Fully mask credential fields in Python mapping ``repr`` output."""
def _sub(match: re.Match) -> str:
key = match.group("key")
if not _is_python_repr_secret_key(key):
return match.group(0)
single_value = match.group("single_value")
if single_value is not None:
prefix = match.group("single_prefix") or ""
quote = "'"
value = single_value
else:
prefix = match.group("double_prefix") or ""
quote = '"'
value = match.group("double_value")
# Mapping repr can contain code-shaped fixture values too. Preserve
# programmatic env lookups just like the ENV/JSON/YAML passes do.
if _ENV_LOOKUP_VALUE_RE.match(value):
return match.group(0)
# Do not retain head/tail characters here: escaped repr atoms can cross
# a slicing boundary and leave an unescaped quote behind. A full mask is
# parseable for both str and bytes values and leaks no opaque bytes.
return f"'{key}'{match.group('sep')}{prefix}{quote}***{quote}"
return _PYTHON_REPR_FIELD_RE.sub(_sub, text)
def _redact_python_diagnostic_repr_fields(text: str) -> str:
"""Mask repr fields only on pytest/error lines in source-preserving output."""
lines = text.splitlines(keepends=True)
for index, line in enumerate(lines):
ending = ""
body_line = line
if line.endswith("\r\n"):
body_line, ending = line[:-2], "\r\n"
elif line.endswith("\n") or line.endswith("\r"):
body_line, ending = line[:-1], line[-1:]
match = _PYTEST_DIAGNOSTIC_LINE_RE.match(body_line)
if match is None:
match = _PYTHON_EXCEPTION_LINE_RE.match(body_line)
if match is not None:
lines[index] = (
match.group("prefix")
+ _redact_python_repr_fields(match.group("body"))
+ ending
)
return "".join(lines)
def _redact_query_string(query: str) -> str:
"""Replace values of sensitive ``k=v&k=v`` params with ``***``; others pass through."""
if not query:
@@ -624,6 +735,11 @@ def _redact_assignments(text: str) -> str:
text = _JSON_FIELD_RE.sub(
_assignment_sub(lambda g: f'{g[0]}: "{_mask_token(g[1])}"', check_keyword=False), text)
# Python mapping repr fields ({'API_KEY': '…'}): single-quoted, so the JSON rule
# above never sees them — the traceback / pytest-introspection leak shape.
if ":" in text and "'" in text:
text = _redact_python_repr_fields(text)
# YAML after JSON: quoted values are handled there (_YAML_ASSIGN_RE skips quotes).
if ":" in text and "://" not in text:
text = _YAML_ASSIGN_RE.sub(
@@ -670,6 +786,10 @@ def redact_sensitive_text(text: str, *, force: bool = False, code_file: bool = F
Every regex sits behind a cheap substring gate that its pattern requires,
so the gates are never false-negative.
Set code_file=True to also skip the Python-repr mapping pass (``{'API_KEY': '…'}``
fixtures in source); pytest/exception diagnostic lines get a narrow pass in
redact_terminal_output instead.
Set file_read=True for file *content* returned to the agent (read_file / search_files / cat). The old
mask looked like a real-but-truncated key, so an agent reading it from config.yaml and writing it back
silently corrupted the stored credential into a dead 13-char value → 401 (issue #35519). The sentinel is
@@ -892,7 +1012,13 @@ def redact_terminal_output(output: str, command: str | None = None, *, force: bo
if not output:
return output
code_file = not (is_env_dump_command(command) or _command_reads_secret_file(command))
return redact_sensitive_text(output, force=force, code_file=code_file)
redacted = redact_sensitive_text(output, force=force, code_file=code_file)
# Source-preserving output still gets the Python-repr pass on high-confidence
# diagnostic lines (pytest ``E `` introspection, final exception lines): that is
# where {'BRAVE_API_KEY': '…'} leaks, not in source dumps.
if code_file and (force or _redact_enabled()) and ":" in redacted and "'" in redacted:
redacted = _redact_python_diagnostic_repr_fields(redacted)
return redacted
# --- Prefix pre-screen: derived from _PREFIX_PATTERNS so a new prefix can't
+131
View File
@@ -1,5 +1,6 @@
"""Tests for agent.redact -- secret masking in logs and output."""
import ast
import logging
import pytest
@@ -290,6 +291,76 @@ class TestJsonFields:
assert result == text
class TestPythonReprFields:
@pytest.mark.parametrize(
"key",
[
"BRAVE_API_KEY",
"SERVICE_TOKEN",
"DB_PASSWORD",
"api_key",
"access_token",
"client_secret",
"id_token",
"private_key",
],
)
def test_secret_field_in_nested_python_repr_is_redacted(self, key):
secret = f"opaque-{key.lower()}-value-1234567890"
text = f"kwargs={{'env': {{'{key}': '{secret}'}}}}"
result = redact_sensitive_text(text, force=True)
assert secret not in result
assert f"'{key}': '***'" in result
@pytest.mark.parametrize(
"key",
["TOKEN_COUNT", "AUTH_METHOD", "PASSWORD_POLICY", "SECRET_NAME", "CREDENTIAL_TYPE"],
)
def test_uppercase_metadata_field_is_unchanged(self, key):
text = f"{{'{key}': 'public-metadata-value'}}"
assert redact_sensitive_text(text, force=True) == text
@pytest.mark.parametrize(
"value",
[
"apostrophe-near-head-'1234567890",
"apostrophe-near-tail-1234567890'",
b"bytes-apostrophe-near-head-'1234567890",
b"bytes-apostrophe-near-tail-1234567890'",
],
)
def test_escaped_repr_value_remains_parseable(self, value):
text = repr({"API_KEY": value})
result = redact_sensitive_text(text, force=True)
parsed = ast.literal_eval(result)
assert parsed["API_KEY"] == (b"***" if isinstance(value, bytes) else "***")
def test_double_quoted_repr_value_is_redacted(self):
secret = "opaque-double-quoted-value-1234567890"
text = f"payload={{'SERVICE_TOKEN': \"{secret}\"}}"
result = redact_sensitive_text(text, force=True)
assert secret not in result
assert "'SERVICE_TOKEN': \"***\"" in result
def test_programmatic_env_lookup_is_preserved(self):
text = "{'API_KEY': \"os.getenv('OPENAI_API_KEY')\"}"
assert redact_sensitive_text(text, force=True) == text
def test_non_secret_python_repr_field_is_unchanged(self):
text = "{'model': 'gpt-5', 'token_count': '123'}"
assert redact_sensitive_text(text, force=True) == text
def test_code_file_preserves_secret_shaped_fixture(self):
text = "CONFIG = {'BRAVE_API_KEY': 'fixture-value-1234567890'}"
assert redact_sensitive_text(text, force=True, code_file=True) == text
class TestAuthHeaders:
@@ -1051,6 +1122,66 @@ class TestTerminalOutputRedaction:
red = redact_terminal_output(out, "printenv")
assert "zzzopaque1234567890abcdef" in red
@pytest.mark.parametrize(
"command",
[
"python -m pytest",
"uv run pytest",
"poetry run pytest",
"coverage run -m pytest",
"PYTHONPATH=. pytest",
"/usr/bin/env pytest",
"bash -lc 'pytest tests/agent/test_redact.py'",
],
)
def test_pytest_diagnostic_masks_python_repr_secret_field(self, command):
from agent.redact import redact_terminal_output
secret = "opaque-brave-value-1234567890"
out = f"E kwargs={{'env': {{'BRAVE_API_KEY': '{secret}'}}}}"
red = redact_terminal_output(out, command, force=True)
assert secret not in red
assert "'BRAVE_API_KEY': '***'" in red
def test_exception_line_masks_python_repr_secret_field(self):
from agent.redact import redact_terminal_output
secret = "opaque-exception-value-1234567890"
out = f"RuntimeError: payload={{'API_KEY': '{secret}'}}"
red = redact_terminal_output(out, "python app.py", force=True)
assert secret not in red
assert "'API_KEY': '***'" in red
def test_source_dump_preserves_python_repr_fixture(self):
from agent.redact import redact_terminal_output
out = "CONFIG = {'BRAVE_API_KEY': 'fixture-value-1234567890'}"
assert redact_terminal_output(out, "cat config.py", force=True) == out
assert redact_terminal_output(out, "python dump_source.py", force=True) == out
def test_pytest_source_line_preserves_python_repr_fixture(self):
from agent.redact import redact_terminal_output
source = " CONFIG = {'BRAVE_API_KEY': 'fixture-value-1234567890'}"
out = source + "\nE assert False"
red = redact_terminal_output(out, "uv run pytest", force=True)
assert source in red
def test_disabled_pytest_diagnostic_passes_through(self, monkeypatch):
from agent.redact import redact_terminal_output
monkeypatch.setattr("agent.redact._REDACT_ENABLED", False)
secret = "opaque-disabled-value-1234567890"
out = f"E payload={{'API_KEY': '{secret}'}}"
assert redact_terminal_output(out, "uv run pytest") == out
class TestFileReadNonReusableRedaction:
"""#35519: prefix-matched credentials in FILE CONTENT (read_file /