fix(update): conservative outcomes + serve-ledger coverage for fresh restart recovery
Salvage adjustments to PR #94392 per review: - Narrow the supervisor claim to the systemd-VERIFIED path only. The fresh recovery child now probes 'systemctl --user is-active' after each relaunch; only an observed-active systemd unit is reported 'verified'. A relaunch that merely exited 0 is labelled 'relaunch_attempted', never counts as supervisor coverage, and never clears gateway_fleet_restart_incomplete. - Serve-owned runtimes (serve/dashboard entries from the spawn ledger, per the update_inventory serve collector) are no longer silently skipped: the recovery pass records them (and manual gateways) as skipped-with-reason in the recovery result and the persisted update receipt. - Receipt fresh_recovery persists the conservative vocabulary (requested/verified/relaunch_attempted/failed/skipped); 'succeeded' is gone. - Added an end-to-end test that drives the real recovery module in a genuinely fresh interpreter (sitecustomize shim intercepts the grandchild 'gateway restart' and systemctl probes).
This commit is contained in:
@@ -5083,6 +5083,7 @@ _LAZY_COMMAND_EXPORTS = {
|
||||
"_format_concurrent_instances_message",
|
||||
"_format_time_ago",
|
||||
"_gateway_service_matches_profile",
|
||||
"_gateway_recovery_partition",
|
||||
"_gateway_restart_recovery_profiles",
|
||||
"_handoff_reapable_backend_pids",
|
||||
"_ledger_reapable_backend_pids",
|
||||
|
||||
+143
-58
@@ -5615,36 +5615,89 @@ def _gateway_service_matches_profile(profile: str, service: object) -> bool:
|
||||
}
|
||||
|
||||
|
||||
def _gateway_restart_recovery_profiles(
|
||||
def _gateway_recovery_partition(
|
||||
plan, *, skip_profiles: set[str] | None = None
|
||||
) -> list[str]:
|
||||
"""Return supervised gateway profiles that a fresh process may restart.
|
||||
) -> tuple[dict[str, str], list[dict]]:
|
||||
"""Partition pre-update runtimes into fresh-restart candidates and skips.
|
||||
|
||||
The update inventory is captured before the checkout changes. It is the
|
||||
only safe source here: re-importing ``hermes_cli.gateway`` in the failing
|
||||
interpreter is exactly what can raise the original ``ImportError``. Manual
|
||||
gateways are intentionally excluded because no supervisor is available to
|
||||
bring them back after a stop.
|
||||
interpreter is exactly what can raise the original ``ImportError``.
|
||||
|
||||
Returns ``(candidates, skipped)`` where ``candidates`` maps profile →
|
||||
supervisor for supervised gateway runtimes the fresh process may restart,
|
||||
and ``skipped`` lists every other inventoried runtime the recovery pass
|
||||
deliberately does NOT touch, each with an explicit reason. Nothing from
|
||||
the spawn ledger may vanish from the recovery pass silently: manual
|
||||
gateways have no relaunch authority, and serve/dashboard runtimes (the
|
||||
``update_inventory`` serve collector) are owned by the Desktop app or a
|
||||
human terminal, not by this recovery boundary.
|
||||
"""
|
||||
profiles: set[str] = set()
|
||||
skip_profiles = skip_profiles or set()
|
||||
candidates: dict[str, str] = {}
|
||||
skipped: list[dict] = []
|
||||
try:
|
||||
for runtime in getattr(plan, "runtimes", ()) or ():
|
||||
if (
|
||||
getattr(runtime, "kind", None) == "gateway"
|
||||
and getattr(runtime, "supervisor", None) in _FRESH_RESTART_SUPERVISORS
|
||||
):
|
||||
profile = getattr(runtime, "profile", None)
|
||||
if isinstance(profile, str) and profile and profile not in skip_profiles:
|
||||
profiles.add(profile)
|
||||
kind = getattr(runtime, "kind", None)
|
||||
profile = getattr(runtime, "profile", None)
|
||||
supervisor = getattr(runtime, "supervisor", None)
|
||||
if not isinstance(profile, str) or not profile:
|
||||
continue
|
||||
if kind == "gateway":
|
||||
if profile in skip_profiles:
|
||||
continue
|
||||
if supervisor in _FRESH_RESTART_SUPERVISORS:
|
||||
candidates.setdefault(profile, str(supervisor))
|
||||
else:
|
||||
skipped.append(
|
||||
{
|
||||
"profile": profile,
|
||||
"kind": "gateway",
|
||||
"supervisor": str(supervisor),
|
||||
"reason": (
|
||||
"manual gateway has no supervisor relaunch"
|
||||
" authority; left running for explicit operator"
|
||||
" restart"
|
||||
),
|
||||
}
|
||||
)
|
||||
elif kind in ("serve", "dashboard"):
|
||||
if supervisor == "desktop":
|
||||
reason = (
|
||||
"desktop app owns and respawns this serve backend;"
|
||||
" the recovery pass must not restart it out from under"
|
||||
" its supervisor"
|
||||
)
|
||||
else:
|
||||
reason = (
|
||||
"manually launched serve/dashboard has no relaunch"
|
||||
" authority; left running for explicit operator"
|
||||
" restart"
|
||||
)
|
||||
skipped.append(
|
||||
{
|
||||
"profile": profile,
|
||||
"kind": str(kind),
|
||||
"supervisor": str(supervisor),
|
||||
"reason": reason,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not prepare fresh gateway restart profiles: %s", exc)
|
||||
return sorted(profiles)
|
||||
return candidates, skipped
|
||||
|
||||
|
||||
def _gateway_restart_recovery_profiles(
|
||||
plan, *, skip_profiles: set[str] | None = None
|
||||
) -> list[str]:
|
||||
"""Return supervised gateway profiles that a fresh process may restart."""
|
||||
candidates, _ = _gateway_recovery_partition(plan, skip_profiles=skip_profiles)
|
||||
return sorted(candidates)
|
||||
|
||||
|
||||
def _recover_gateway_restart_after_abort(
|
||||
plan, *, gateway_mode: bool, skip_profiles: set[str] | None = None
|
||||
) -> dict[str, list[str]]:
|
||||
) -> dict[str, list]:
|
||||
"""Retry supervised gateway restarts from a clean Python process.
|
||||
|
||||
``hermes update`` normally performs the fleet restart in the interpreter
|
||||
@@ -5658,12 +5711,38 @@ def _recover_gateway_restart_after_abort(
|
||||
Only profiles classified as supervisor-owned by the pre-update inventory
|
||||
are handed off. A manual gateway must remain running and be reported for
|
||||
explicit operator action rather than being killed without a relaunch
|
||||
authority. The returned protocol is persisted in the update receipt so
|
||||
operators can distinguish a spawn failure from a per-profile failure.
|
||||
authority; serve/dashboard runtimes from the spawn ledger are likewise
|
||||
recorded as skipped with a reason instead of vanishing from the pass.
|
||||
The returned protocol is persisted in the update receipt so operators can
|
||||
distinguish a spawn failure from a per-profile failure.
|
||||
|
||||
Outcome honesty: ``verified`` means the fresh child independently observed
|
||||
the profile's systemd unit active after the relaunch. A zero exit from
|
||||
``gateway restart`` alone is NOT observed proof that the new code
|
||||
generation is serving, so those outcomes are reported as
|
||||
``relaunch_attempted`` and never claim supervisor coverage.
|
||||
"""
|
||||
profiles = _gateway_restart_recovery_profiles(plan, skip_profiles=skip_profiles)
|
||||
candidates, skipped = _gateway_recovery_partition(
|
||||
plan, skip_profiles=skip_profiles
|
||||
)
|
||||
profiles = sorted(candidates)
|
||||
if not profiles:
|
||||
return {"requested": [], "succeeded": [], "failed": []}
|
||||
return {
|
||||
"requested": [],
|
||||
"verified": [],
|
||||
"relaunch_attempted": [],
|
||||
"failed": [],
|
||||
"skipped": skipped,
|
||||
}
|
||||
|
||||
def _all_failed() -> dict[str, list]:
|
||||
return {
|
||||
"requested": profiles,
|
||||
"verified": [],
|
||||
"relaunch_attempted": [],
|
||||
"failed": profiles,
|
||||
"skipped": skipped,
|
||||
}
|
||||
|
||||
command = [
|
||||
sys.executable,
|
||||
@@ -5685,7 +5764,7 @@ def _recover_gateway_restart_after_abort(
|
||||
systemd_run = shutil.which("systemd-run")
|
||||
if not systemd_run:
|
||||
logger.warning("Cannot isolate fresh gateway recovery from the gateway cgroup")
|
||||
return {"requested": profiles, "succeeded": [], "failed": profiles}
|
||||
return _all_failed()
|
||||
command = [
|
||||
systemd_run,
|
||||
"--user",
|
||||
@@ -5697,7 +5776,7 @@ def _recover_gateway_restart_after_abort(
|
||||
]
|
||||
|
||||
kwargs = {
|
||||
"input": json.dumps({"profiles": profiles}),
|
||||
"input": json.dumps({"profiles": profiles, "supervisors": candidates}),
|
||||
"capture_output": True,
|
||||
"text": True,
|
||||
"encoding": "utf-8",
|
||||
@@ -5718,39 +5797,52 @@ def _recover_gateway_restart_after_abort(
|
||||
result = subprocess.run(command, **kwargs)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
logger.warning("Fresh gateway restart recovery failed: %s", exc)
|
||||
return {"requested": profiles, "succeeded": [], "failed": profiles}
|
||||
return _all_failed()
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.warning("Fresh gateway restart recovery exited %s", result.returncode)
|
||||
return {"requested": profiles, "succeeded": [], "failed": profiles}
|
||||
return _all_failed()
|
||||
|
||||
try:
|
||||
recovery_result = json.loads(result.stdout or "")
|
||||
succeeded = recovery_result.get("succeeded")
|
||||
verified = recovery_result.get("verified")
|
||||
relaunch_attempted = recovery_result.get("relaunch_attempted")
|
||||
failed = recovery_result.get("failed")
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
logger.warning("Fresh gateway restart recovery returned invalid JSON")
|
||||
return {"requested": profiles, "succeeded": [], "failed": profiles}
|
||||
return _all_failed()
|
||||
|
||||
buckets = (verified, relaunch_attempted, failed)
|
||||
reported: list[str] = []
|
||||
if all(isinstance(bucket, list) for bucket in buckets):
|
||||
reported = [*verified, *relaunch_attempted, *failed]
|
||||
if (
|
||||
not isinstance(succeeded, list)
|
||||
or not isinstance(failed, list)
|
||||
or any(not isinstance(profile, str) for profile in [*succeeded, *failed])
|
||||
or set(succeeded) != set(profiles)
|
||||
or failed
|
||||
not all(isinstance(bucket, list) for bucket in buckets)
|
||||
or any(not isinstance(profile, str) for profile in reported)
|
||||
or set(reported) != set(profiles)
|
||||
or len(reported) != len(set(reported))
|
||||
):
|
||||
logger.warning("Fresh gateway restart recovery returned incomplete profiles")
|
||||
return {
|
||||
"requested": profiles,
|
||||
"succeeded": succeeded if isinstance(succeeded, list) else [],
|
||||
"failed": failed if isinstance(failed, list) else profiles,
|
||||
}
|
||||
return _all_failed()
|
||||
|
||||
print(
|
||||
" ✓ Retried supervised gateway restart(s) in a fresh process: "
|
||||
+ ", ".join(profiles)
|
||||
)
|
||||
return {"requested": profiles, "succeeded": succeeded, "failed": []}
|
||||
if verified:
|
||||
print(
|
||||
" ✓ Restarted supervised gateway(s) in a fresh process"
|
||||
" (systemd-verified active): " + ", ".join(sorted(verified))
|
||||
)
|
||||
if relaunch_attempted:
|
||||
print(
|
||||
" ⚠ Relaunch attempted in a fresh process but not"
|
||||
" supervisor-verified (check these gateways manually): "
|
||||
+ ", ".join(sorted(relaunch_attempted))
|
||||
)
|
||||
return {
|
||||
"requested": profiles,
|
||||
"verified": sorted(verified),
|
||||
"relaunch_attempted": sorted(relaunch_attempted),
|
||||
"failed": sorted(failed),
|
||||
"skipped": skipped,
|
||||
}
|
||||
|
||||
|
||||
def _warn_gateway_restart_phase_aborted(exc: BaseException, pids) -> None:
|
||||
@@ -8802,16 +8894,14 @@ def _cmd_update_impl(args, gateway_mode: bool):
|
||||
gateway_mode=gateway_mode,
|
||||
skip_profiles=_already_restarted_profiles,
|
||||
)
|
||||
_recovery_profiles = list(_recovery_result.get("requested") or [])
|
||||
_recovery_succeeded = set(_recovery_result.get("succeeded") or [])
|
||||
_recovered = not _recovery_profiles or (
|
||||
_recovery_succeeded == set(_recovery_profiles)
|
||||
and not _recovery_result.get("failed")
|
||||
)
|
||||
if _recovery_succeeded:
|
||||
# Only systemd-VERIFIED outcomes may claim supervisor coverage.
|
||||
# A relaunch that merely exited 0 ("relaunch_attempted") was never
|
||||
# observed by the code and must not clear the incomplete flag.
|
||||
_recovery_verified = set(_recovery_result.get("verified") or [])
|
||||
if _recovery_verified:
|
||||
relaunched_profiles.extend(
|
||||
profile
|
||||
for profile in sorted(_recovery_succeeded)
|
||||
for profile in sorted(_recovery_verified)
|
||||
if profile not in relaunched_profiles
|
||||
)
|
||||
_planned_gateway_runtimes = [
|
||||
@@ -8823,18 +8913,13 @@ def _cmd_update_impl(args, gateway_mode: bool):
|
||||
_planned_gateway_profiles = {
|
||||
runtime.profile for runtime in _planned_gateway_runtimes
|
||||
}
|
||||
_covered_gateway_profiles = _already_restarted_profiles | set(
|
||||
_recovery_succeeded
|
||||
)
|
||||
_all_gateway_profiles_are_safe_to_recover = all(
|
||||
runtime.profile in _already_restarted_profiles
|
||||
or getattr(runtime, "supervisor", None) in _FRESH_RESTART_SUPERVISORS
|
||||
for runtime in _planned_gateway_runtimes
|
||||
_covered_gateway_profiles = (
|
||||
_already_restarted_profiles | _recovery_verified
|
||||
)
|
||||
_recovery_complete = bool(_planned_gateway_profiles) and (
|
||||
_planned_gateway_profiles <= _covered_gateway_profiles
|
||||
and _all_gateway_profiles_are_safe_to_recover
|
||||
and (not _recovery_profiles or _recovered)
|
||||
and not _recovery_result.get("failed")
|
||||
and not _recovery_result.get("relaunch_attempted")
|
||||
)
|
||||
if _recovery_complete:
|
||||
# The fresh child is the recovery terminal result. Leave the
|
||||
|
||||
@@ -113,17 +113,26 @@ class UpdateReceipt:
|
||||
"phase_error": phase_error,
|
||||
}
|
||||
if fresh_recovery is not None:
|
||||
result["fresh_recovery"] = {
|
||||
"requested": [
|
||||
str(profile) for profile in fresh_recovery.get("requested", [])
|
||||
],
|
||||
"succeeded": [
|
||||
str(profile) for profile in fresh_recovery.get("succeeded", [])
|
||||
],
|
||||
"failed": [
|
||||
str(profile) for profile in fresh_recovery.get("failed", [])
|
||||
],
|
||||
# Conservative outcome vocabulary: "verified" is the only bucket
|
||||
# allowed to claim supervisor coverage; "relaunch_attempted" means
|
||||
# the relaunch exited 0 without independent supervisor
|
||||
# observation. "skipped" preserves runtimes (manual gateways,
|
||||
# serve/dashboard entries) the pass deliberately did not touch.
|
||||
persisted: dict[str, Any] = {
|
||||
key: [str(profile) for profile in fresh_recovery.get(key, [])]
|
||||
for key in ("requested", "verified", "relaunch_attempted", "failed")
|
||||
}
|
||||
persisted["skipped"] = [
|
||||
{
|
||||
"profile": str(entry.get("profile", "")),
|
||||
"kind": str(entry.get("kind", "")),
|
||||
"supervisor": str(entry.get("supervisor", "")),
|
||||
"reason": str(entry.get("reason", "")),
|
||||
}
|
||||
for entry in fresh_recovery.get("skipped", [])
|
||||
if isinstance(entry, dict)
|
||||
]
|
||||
result["fresh_recovery"] = persisted
|
||||
self.data["gateway_restart"] = result
|
||||
|
||||
def finalize(self, outcome: str) -> None:
|
||||
|
||||
@@ -6,6 +6,19 @@ itself and launches the regular per-profile gateway command in a new
|
||||
interpreter. It is used only after the in-process restart phase has raised, so
|
||||
that the recovery path cannot inherit the stale ``sys.modules`` graph that
|
||||
caused the failure.
|
||||
|
||||
Outcome vocabulary (deliberately conservative):
|
||||
|
||||
- ``verified`` — the relaunch command exited 0 AND the profile's
|
||||
systemd unit was independently observed ``active`` afterwards. This is the
|
||||
only outcome that may claim supervisor coverage.
|
||||
- ``relaunch_attempted`` — the relaunch command exited 0 but no independent
|
||||
supervisor observation was possible (non-systemd supervisor, ``systemctl``
|
||||
missing, or the unit probe was inconclusive). ``rc == 0`` from
|
||||
``gateway restart`` is not proof that the new code generation is running,
|
||||
so this outcome must never be treated as verified coverage.
|
||||
- ``failed`` — the relaunch command errored, timed out, or exited
|
||||
non-zero.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,15 +27,18 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
_RECOVERY_ENV = "HERMES_UPDATE_RESTART_RECOVERY"
|
||||
_GATEWAY_MARKERS = ("_HERMES_GATEWAY", "HERMES_GATEWAY", "HERMES_GATEWAY_MODE")
|
||||
_PROFILE_RESTART_TIMEOUT = 90
|
||||
_VERIFY_TIMEOUT = 15
|
||||
_PROFILE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
|
||||
_SUPERVISOR_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,31}$")
|
||||
|
||||
|
||||
def _profile_command(profile: str) -> list[str]:
|
||||
@@ -78,8 +94,57 @@ def _run_profile_restart(
|
||||
return getattr(result, "returncode", 1) == 0
|
||||
|
||||
|
||||
def _systemd_unit_candidates(profile: str) -> tuple[str, ...]:
|
||||
"""Unit names the existing systemd gateway lifecycle produces per profile."""
|
||||
if profile == "default":
|
||||
return (
|
||||
"hermes-gateway.service",
|
||||
"gateway.service",
|
||||
"gateway-default.service",
|
||||
)
|
||||
return (
|
||||
f"hermes-gateway-{profile}.service",
|
||||
f"gateway-{profile}.service",
|
||||
)
|
||||
|
||||
|
||||
def _systemd_verified_active(profile: str, *, run: Callable[..., Any]) -> bool:
|
||||
"""Return True only when systemd itself reports the profile's unit active.
|
||||
|
||||
This is the observation that separates ``verified`` from
|
||||
``relaunch_attempted``. Any failure here (no ``systemctl``, probe error,
|
||||
unit not ``active``) means we could NOT verify — never that the restart
|
||||
failed.
|
||||
"""
|
||||
systemctl = shutil.which("systemctl")
|
||||
if not systemctl:
|
||||
return False
|
||||
for unit in _systemd_unit_candidates(profile):
|
||||
try:
|
||||
result = run(
|
||||
[systemctl, "--user", "is-active", unit],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=False,
|
||||
timeout=_VERIFY_TIMEOUT,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
continue
|
||||
if (
|
||||
getattr(result, "returncode", 1) == 0
|
||||
and (getattr(result, "stdout", "") or "").strip() == "active"
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def restart_profiles(
|
||||
profiles: Iterable[str], *, run: Callable[..., Any] = subprocess.run
|
||||
profiles: Iterable[str],
|
||||
*,
|
||||
supervisors: Mapping[str, str] | None = None,
|
||||
run: Callable[..., Any] = subprocess.run,
|
||||
) -> dict[str, list[str]]:
|
||||
"""Restart the supplied profiles and return per-profile terminal results.
|
||||
|
||||
@@ -87,21 +152,38 @@ def restart_profiles(
|
||||
supervisor. Manual gateways are intentionally excluded before this module
|
||||
is called: killing one without a relaunch authority would turn stale code
|
||||
into an outage.
|
||||
|
||||
A profile only lands in ``verified`` when its supervisor is systemd and
|
||||
``systemctl --user is-active`` independently confirms the unit after the
|
||||
relaunch command succeeded. Every other zero-exit relaunch is reported as
|
||||
``relaunch_attempted`` — the code cannot observe supervisor coverage for
|
||||
those paths and must not claim it.
|
||||
"""
|
||||
supervisors = supervisors or {}
|
||||
normalized = sorted(
|
||||
{profile for profile in profiles if isinstance(profile, str) and profile}
|
||||
)
|
||||
succeeded: list[str] = []
|
||||
verified: list[str] = []
|
||||
relaunch_attempted: list[str] = []
|
||||
failed: list[str] = []
|
||||
for profile in normalized:
|
||||
if _run_profile_restart(profile, run=run):
|
||||
succeeded.append(profile)
|
||||
else:
|
||||
if not _run_profile_restart(profile, run=run):
|
||||
failed.append(profile)
|
||||
return {"succeeded": succeeded, "failed": failed}
|
||||
continue
|
||||
if supervisors.get(profile) == "systemd" and _systemd_verified_active(
|
||||
profile, run=run
|
||||
):
|
||||
verified.append(profile)
|
||||
else:
|
||||
relaunch_attempted.append(profile)
|
||||
return {
|
||||
"verified": verified,
|
||||
"relaunch_attempted": relaunch_attempted,
|
||||
"failed": failed,
|
||||
}
|
||||
|
||||
|
||||
def _parse_payload(stream) -> list[str]:
|
||||
def _parse_payload(stream) -> tuple[list[str], dict[str, str]]:
|
||||
payload = json.load(stream)
|
||||
profiles = payload.get("profiles") if isinstance(payload, dict) else None
|
||||
if not isinstance(profiles, list):
|
||||
@@ -111,7 +193,19 @@ def _parse_payload(stream) -> list[str]:
|
||||
for profile in profiles
|
||||
):
|
||||
raise ValueError("recovery profiles contain an invalid profile id")
|
||||
return profiles
|
||||
raw_supervisors = payload.get("supervisors") if isinstance(payload, dict) else None
|
||||
supervisors: dict[str, str] = {}
|
||||
if raw_supervisors is not None:
|
||||
if not isinstance(raw_supervisors, dict) or any(
|
||||
not isinstance(profile, str)
|
||||
or not isinstance(supervisor, str)
|
||||
or not _PROFILE_ID_RE.fullmatch(profile)
|
||||
or not _SUPERVISOR_RE.fullmatch(supervisor)
|
||||
for profile, supervisor in raw_supervisors.items()
|
||||
):
|
||||
raise ValueError("recovery supervisors map is invalid")
|
||||
supervisors = dict(raw_supervisors)
|
||||
return profiles, supervisors
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
@@ -126,10 +220,19 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.error("this command is an internal update-recovery entry point")
|
||||
|
||||
try:
|
||||
profiles = _parse_payload(sys.stdin)
|
||||
result = restart_profiles(profiles)
|
||||
profiles, supervisors = _parse_payload(sys.stdin)
|
||||
result = restart_profiles(profiles, supervisors=supervisors)
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
print(json.dumps({"error": str(exc), "succeeded": [], "failed": []}))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"error": str(exc),
|
||||
"verified": [],
|
||||
"relaunch_attempted": [],
|
||||
"failed": [],
|
||||
}
|
||||
)
|
||||
)
|
||||
return 2
|
||||
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
|
||||
Reference in New Issue
Block a user