Harden MATLAB parity retries and add parity runbook

iahncajigas · iahncajigas · commit df13d57a31ca · 2026-02-26T14:29:53.000-05:00
diff --git a/.github/workflows/matlab-parity-gate.yml b/.github/workflows/matlab-parity-gate.yml
@@ -25,8 +25,11 @@ jobs:
       ACTIONS_RUNNER_SVC: "1"
       NSTAT_MATLAB_EXTRA_ARGS: -maca64 -nodisplay -noFigureWindows -softwareopengl
       NSTAT_FORCE_M_HELP_SCRIPTS: "1"
+      NSTAT_MATLAB_TOPIC_MAX_ATTEMPTS: "2"
       NSTAT_PARITY_RETRY_TIMEOUT_BLOCKS: "1"
       NSTAT_PARITY_TIMEOUT_RETRY_BLOCKS: timeout_front
+      NSTAT_PARITY_RETRY_RECOVERABLE_BLOCKS: "1"
+      NSTAT_PARITY_RECOVERABLE_RETRY_BLOCKS: graphics_mid,heavy_tail
     steps:
       - name: Prepare runner directories
         run: |
diff --git a/python/README.md b/python/README.md
@@ -86,8 +86,12 @@ Notes:
   `python/tools/run_parity_ladder.sh core_smoke timeout_front`.
 - Ladder writes retry telemetry to `python/reports/parity_retry_summary.json` (block, attempt count, retry reason, timeout-topic list).
 - Retry behavior is controlled by `NSTAT_PARITY_RETRY_TIMEOUT_BLOCKS` and `NSTAT_PARITY_TIMEOUT_RETRY_BLOCKS`.
+- Set `NSTAT_MATLAB_TOPIC_MAX_ATTEMPTS=2` to retry per-topic MATLAB timeouts/crashes once before failing.
+- Set `NSTAT_PARITY_RETRY_RECOVERABLE_BLOCKS=1` and `NSTAT_PARITY_RECOVERABLE_RETRY_BLOCKS` to retry block failures caused by recoverable MATLAB failures (timeouts/crash signatures).
 - Preflight topic selection can be overridden with `NSTAT_PARITY_PREFLIGHT_STAGEB_TOPICS`.
 
+See `python/docs/parity_runbook.rst` for the exact locally validated parity command set.
+
 Use targeted blocks to debug delays locally before running remote CI:
 
 ```bash
diff --git a/python/docs/index.rst b/python/docs/index.rst
@@ -8,3 +8,4 @@ Standalone Python port of nSTAT with MATLAB-help topic coverage and executable n
 
    api
    help_topics
+   parity_runbook
diff --git a/python/docs/parity_runbook.rst b/python/docs/parity_runbook.rst
@@ -0,0 +1,71 @@
+Parity Runbook
+==============
+
+Use this runbook for reproducible local/CI MATLAB parity checks on this machine.
+
+Validated environment
+---------------------
+
+- MATLAB binary: ``/Applications/MATLAB_R2025b.app/bin/matlab``
+- MATLAB args: ``-maca64 -nodisplay -noFigureWindows -softwareopengl``
+- Runner service mode: ``ACTIONS_RUNNER_SVC=1``
+- Force ``.m`` help scripts: ``NSTAT_FORCE_M_HELP_SCRIPTS=1``
+- Per-topic MATLAB retries: ``NSTAT_MATLAB_TOPIC_MAX_ATTEMPTS=2``
+- Ladder timeout-only retry: ``NSTAT_PARITY_RETRY_TIMEOUT_BLOCKS=1``
+- Ladder recoverable retry: ``NSTAT_PARITY_RETRY_RECOVERABLE_BLOCKS=1``
+
+Preflight + staged parity
+-------------------------
+
+Run Stage A + selected Stage B preflight:
+
+.. code-block:: bash
+
+   ACTIONS_RUNNER_SVC=1 \
+   NSTAT_FORCE_M_HELP_SCRIPTS=1 \
+   NSTAT_MATLAB_EXTRA_ARGS='-maca64 -nodisplay -noFigureWindows -softwareopengl' \
+   NSTAT_MATLAB_TOPIC_MAX_ATTEMPTS=2 \
+   python/tools/run_parity_preflight.sh
+
+Run the staged ladder (core -> timeout -> graphics -> heavy-tail):
+
+.. code-block:: bash
+
+   ACTIONS_RUNNER_SVC=1 \
+   NSTAT_FORCE_M_HELP_SCRIPTS=1 \
+   NSTAT_MATLAB_EXTRA_ARGS='-maca64 -nodisplay -noFigureWindows -softwareopengl' \
+   NSTAT_MATLAB_TOPIC_MAX_ATTEMPTS=2 \
+   NSTAT_PARITY_RETRY_TIMEOUT_BLOCKS=1 \
+   NSTAT_PARITY_TIMEOUT_RETRY_BLOCKS=timeout_front \
+   NSTAT_PARITY_RETRY_RECOVERABLE_BLOCKS=1 \
+   NSTAT_PARITY_RECOVERABLE_RETRY_BLOCKS='graphics_mid,heavy_tail' \
+   python/tools/run_parity_ladder.sh core_smoke timeout_front graphics_mid heavy_tail
+
+Full Stage C gate
+-----------------
+
+Run full-suite parity gate report:
+
+.. code-block:: bash
+
+   ACTIONS_RUNNER_SVC=1 \
+   NSTAT_FORCE_M_HELP_SCRIPTS=1 \
+   NSTAT_MATLAB_EXTRA_ARGS='-maca64 -nodisplay -noFigureWindows -softwareopengl' \
+   NSTAT_MATLAB_TOPIC_MAX_ATTEMPTS=2 \
+   python3 python/tools/verify_python_vs_matlab_similarity.py \
+     --enforce-gate \
+     --matlab-max-attempts 2 \
+     --report-path python/reports/python_vs_matlab_similarity_report.json
+
+Useful outputs
+--------------
+
+- Full report: ``python/reports/python_vs_matlab_similarity_report.json``
+- Ladder retry telemetry: ``python/reports/parity_retry_summary.json``
+- Block reports: ``python/reports/parity_block_*.json``
+- Summary helper:
+
+.. code-block:: bash
+
+   python3 python/tools/summarize_parity_report.py \
+     python/reports/python_vs_matlab_similarity_report.json --json
diff --git a/python/tools/run_parity_ladder.sh b/python/tools/run_parity_ladder.sh
@@ -10,6 +10,8 @@ SET_ACTIONS_RUNNER_SVC="${NSTAT_SET_ACTIONS_RUNNER_SVC:-1}"
 RUNTIME_MULTIPLIER="${NSTAT_PARITY_RUNTIME_MULTIPLIER:-2.5}"
 RETRY_TIMEOUT_BLOCKS="${NSTAT_PARITY_RETRY_TIMEOUT_BLOCKS:-0}"
 TIMEOUT_RETRY_BLOCKS="${NSTAT_PARITY_TIMEOUT_RETRY_BLOCKS:-timeout_front}"
+RETRY_RECOVERABLE_BLOCKS="${NSTAT_PARITY_RETRY_RECOVERABLE_BLOCKS:-1}"
+RECOVERABLE_RETRY_BLOCKS="${NSTAT_PARITY_RECOVERABLE_RETRY_BLOCKS:-graphics_mid,heavy_tail,full_suite}"
 RETRY_SUMMARY_PATH="${NSTAT_PARITY_RETRY_SUMMARY_PATH:-python/reports/parity_retry_summary.json}"
 
 DEFAULT_BLOCKS=(core_smoke timeout_front graphics_mid heavy_tail full_suite)
@@ -40,6 +42,16 @@ block_retry_enabled() {
   return 1
 }
 
+block_recoverable_retry_enabled() {
+  local block="$1"
+  [[ "${RETRY_RECOVERABLE_BLOCKS}" == "1" ]] || return 1
+  local token
+  for token in ${RECOVERABLE_RETRY_BLOCKS//,/ }; do
+    [[ "${token}" == "${block}" ]] && return 0
+  done
+  return 1
+}
+
 is_timeout_only_regression() {
   local report_path="$1"
   "${PYTHON_BIN}" - "${report_path}" <<'PY'
@@ -108,8 +120,52 @@ raise SystemExit(0)
 PY
 }
 
+retryable_failure_topics_csv() {
+  local report_path="$1"
+  "${PYTHON_BIN}" - "${report_path}" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+if not path.exists():
+    raise SystemExit(1)
+payload = json.loads(path.read_text(encoding="utf-8"))
+rows = payload.get("helpfile_similarity", {}).get("rows", [])
+if not rows:
+    raise SystemExit(1)
+failed = [r for r in rows if not bool(r.get("matlab_ok"))]
+if not failed:
+    raise SystemExit(1)
+
+markers = (
+    "matlab_timeout",
+    "matlab is exiting because of fatal error",
+    "fatal error",
+    "mathworkscrashreporter",
+    "crash report has been saved",
+    "libmwhandle_graphics",
+)
+
+def retryable(err: str) -> bool:
+    e = (err or "").strip().lower()
+    if e == "matlab_timeout":
+        return True
+    return any(m in e for m in markers)
+
+if not all(retryable(str(r.get("matlab_error", ""))) for r in failed):
+    raise SystemExit(1)
+
+topics = [str(r.get("topic", "")).strip() for r in failed if str(r.get("topic", "")).strip()]
+if not topics:
+    raise SystemExit(1)
+print(",".join(topics))
+raise SystemExit(0)
+PY
+}
+
 init_retry_summary() {
-  "${PYTHON_BIN}" - "${RETRY_SUMMARY_ABS}" "${RETRY_TIMEOUT_BLOCKS}" "${TIMEOUT_RETRY_BLOCKS}" <<'PY'
+  "${PYTHON_BIN}" - "${RETRY_SUMMARY_ABS}" "${RETRY_TIMEOUT_BLOCKS}" "${TIMEOUT_RETRY_BLOCKS}" "${RETRY_RECOVERABLE_BLOCKS}" "${RECOVERABLE_RETRY_BLOCKS}" <<'PY'
 import json
 import sys
 from datetime import datetime, timezone
@@ -121,6 +177,8 @@ payload = {
     "generated_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
     "retry_timeout_blocks_enabled": sys.argv[2] == "1",
     "timeout_retry_blocks": [b for b in sys.argv[3].replace(",", " ").split() if b],
+    "retry_recoverable_blocks_enabled": sys.argv[4] == "1",
+    "recoverable_retry_blocks": [b for b in sys.argv[5].replace(",", " ").split() if b],
     "events": [],
 }
 path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
@@ -175,6 +233,7 @@ echo "[ladder] matlab args: ${MATLAB_EXTRA_ARGS}"
 echo "[ladder] blocks: ${BLOCKS[*]}"
 echo "[ladder] runtime multiplier: ${RUNTIME_MULTIPLIER} (<=0 disables runtime regression checks)"
 echo "[ladder] retry timeout-only blocks: ${RETRY_TIMEOUT_BLOCKS} (blocks: ${TIMEOUT_RETRY_BLOCKS})"
+echo "[ladder] retry recoverable-failure blocks: ${RETRY_RECOVERABLE_BLOCKS} (blocks: ${RECOVERABLE_RETRY_BLOCKS})"
 echo "[ladder] retry summary path: ${RETRY_SUMMARY_PATH}"
 
 for block in "${BLOCKS[@]}"; do
@@ -186,7 +245,7 @@ for block in "${BLOCKS[@]}"; do
   echo "[ladder] running block: ${block}"
   report_path="${REPO_ROOT}/python/reports/parity_block_${block}.json"
   max_attempts=1
-  if block_retry_enabled "${block}"; then
+  if block_retry_enabled "${block}" || block_recoverable_retry_enabled "${block}"; then
     max_attempts=2
   fi
   attempt=1
@@ -295,6 +354,13 @@ PY
       attempt=$((attempt + 1))
       continue
     fi
+    if [[ "${rc}" -eq 10 ]] && [[ "${attempt}" -lt "${max_attempts}" ]] && retry_topics_csv="$(retryable_failure_topics_csv "${report_path}")"; then
+      echo "[ladder] retrying block ${block} after recoverable MATLAB failures (attempt ${attempt}/${max_attempts}); topics=${retry_topics_csv}"
+      append_retry_summary_event "retry_scheduled" "${block}" "${attempt}" "${max_attempts}" "retry" "${rc}" "recoverable_matlab_failures" "${retry_topics_csv}"
+      warmup_matlab
+      attempt=$((attempt + 1))
+      continue
+    fi
     reason="block_failure"
     if [[ "${rc}" -eq 10 ]]; then
       reason="regression_gate_failure"
diff --git a/python/tools/verify_python_vs_matlab_similarity.py b/python/tools/verify_python_vs_matlab_similarity.py
@@ -66,6 +66,17 @@
     "nSTATPaperExamples",
 }
 DEFAULT_HELP_TOPIC_TIMEOUT_S = 120
+try:
+    DEFAULT_MATLAB_MAX_ATTEMPTS = max(1, int(os.environ.get("NSTAT_MATLAB_TOPIC_MAX_ATTEMPTS", "1")))
+except ValueError:
+    DEFAULT_MATLAB_MAX_ATTEMPTS = 1
+CRASH_ERROR_MARKERS = (
+    "matlab is exiting because of fatal error",
+    "fatal error",
+    "mathworkscrashreporter",
+    "crash report has been saved",
+    "libmwhandle_graphics",
+)
 DEFAULT_TOPIC_TIMEOUT_OVERRIDES: dict[str, int] = {
     "SignalObjExamples": 180,
     "CovariateExamples": 180,
@@ -128,6 +139,32 @@ def _cleanup_runner_matlab_processes() -> None:
     time.sleep(0.5)
 
 
+def _matlab_warmup(timeout_s: int = 90) -> None:
+    if not MATLAB_BIN.exists():
+        return
+    try:
+        _run_matlab_batch_logged("disp(version); exit", timeout_s=timeout_s)
+    except Exception:
+        return
+
+
+def _is_retryable_matlab_failure(payload: dict[str, Any]) -> bool:
+    if bool(payload.get("ok")):
+        return False
+    error = str(payload.get("error", "")).strip()
+    if error == "matlab_timeout":
+        return True
+    combined = " ".join(
+        [
+            error,
+            str(payload.get("error_report", "")),
+            str(payload.get("fallback_error", "")),
+            str(payload.get("fallback_error_report", "")),
+        ]
+    ).lower()
+    return any(marker in combined for marker in CRASH_ERROR_MARKERS)
+
+
 def _kill_process_group(pid: int) -> None:
     try:
         os.killpg(pid, signal.SIGKILL)
@@ -494,6 +531,10 @@ def run_script_path(path: Path, timeout: int, source_label: str | None = None) -
             "end; exit(0);"
         )
 
+        # In runner-service mode, enforce a clean MATLAB process slate before each topic.
+        if _runner_service_mode():
+            _cleanup_runner_matlab_processes()
+
         t0 = time.time()
         try:
             run = _run_matlab_batch_logged(cmd, timeout)
@@ -615,6 +656,7 @@ def _help_similarity(
     topics: list[tuple[str, str]],
     default_timeout_s: int = DEFAULT_HELP_TOPIC_TIMEOUT_S,
     topic_timeout_overrides: dict[str, int] | None = None,
+    matlab_max_attempts: int = DEFAULT_MATLAB_MAX_ATTEMPTS,
 ) -> dict[str, Any]:
     rows: list[dict[str, Any]] = []
 
@@ -649,7 +691,41 @@ def _help_similarity(
 
         py = _run_python_topic(stem)
         timeout_s = topic_timeouts.get(stem, default_timeout_s)
+        ml_attempt_history: list[dict[str, Any]] = []
         ml = _run_matlab_help_script(script_rel, timeout_s=timeout_s)
+        ml_attempt_history.append(
+            {
+                "attempt": 1,
+                "ok": bool(ml.get("ok")),
+                "error": str(ml.get("error", "")),
+                "runtime_s": float(ml.get("runtime_s") or 0.0),
+                "script_used": str(ml.get("script_used", script_rel)),
+            }
+        )
+        attempt = 1
+        while (
+            attempt < matlab_max_attempts
+            and not bool(ml.get("ok"))
+            and _is_retryable_matlab_failure(ml)
+        ):
+            next_attempt = attempt + 1
+            print(
+                f"[help retry {next_attempt}/{matlab_max_attempts}] {stem} "
+                f"after retryable MATLAB failure: {ml.get('error', '')}",
+                flush=True,
+            )
+            _matlab_warmup()
+            ml = _run_matlab_help_script(script_rel, timeout_s=timeout_s)
+            ml_attempt_history.append(
+                {
+                    "attempt": next_attempt,
+                    "ok": bool(ml.get("ok")),
+                    "error": str(ml.get("error", "")),
+                    "runtime_s": float(ml.get("runtime_s") or 0.0),
+                    "script_used": str(ml.get("script_used", script_rel)),
+                }
+            )
+            attempt = next_attempt
 
         if py.get("ok"):
             summary["python_ok"] += 1
@@ -697,6 +773,9 @@ def _help_similarity(
                 "matlab_fallback_script_used": ml.get("fallback_script_used", ""),
                 "matlab_runtime_s": ml.get("runtime_s"),
                 "matlab_timeout_s": timeout_s,
+                "matlab_attempts": len(ml_attempt_history),
+                "matlab_retry_applied": len(ml_attempt_history) > 1,
+                "matlab_attempt_history": ml_attempt_history,
                 "matlab_timeout_snapshot_before_cleanup": ml.get("timeout_process_snapshot_before_cleanup", ""),
                 "matlab_timeout_snapshot_after_cleanup": ml.get("timeout_process_snapshot_after_cleanup", ""),
                 "matlab_runner_service_cleanup": bool(ml.get("runner_service_cleanup", False)),
@@ -878,6 +957,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
         default=[],
         help="Override per-topic MATLAB timeout using TOPIC=SECONDS (repeatable).",
     )
+    parser.add_argument(
+        "--matlab-max-attempts",
+        type=int,
+        default=DEFAULT_MATLAB_MAX_ATTEMPTS,
+        help=(
+            "Maximum MATLAB attempts per help topic for retryable failures "
+            f"(default: {DEFAULT_MATLAB_MAX_ATTEMPTS})."
+        ),
+    )
     parser.add_argument(
         "--report-path",
         default="python/reports/python_vs_matlab_similarity_report.json",
@@ -892,6 +980,9 @@ def main(argv: list[str] | None = None) -> int:
     if args.default_topic_timeout <= 0:
         print("--default-topic-timeout must be positive", file=sys.stderr)
         return 2
+    if args.matlab_max_attempts <= 0:
+        print("--matlab-max-attempts must be positive", file=sys.stderr)
+        return 2
     try:
         requested_topics = _parse_topics_arg(args.topics)
         topics = _resolve_topics(requested_topics)
@@ -910,6 +1001,7 @@ def main(argv: list[str] | None = None) -> int:
         "default_timeout_s": args.default_topic_timeout,
         "topic_timeout_overrides": topic_timeout_overrides,
         "force_m_help_scripts": FORCE_M_HELP_SCRIPTS,
+        "matlab_max_attempts": args.matlab_max_attempts,
     }
 
     print("[class] running Python/MATLAB class checks", flush=True)
@@ -934,6 +1026,7 @@ def main(argv: list[str] | None = None) -> int:
         topics=topics,
         default_timeout_s=args.default_topic_timeout,
         topic_timeout_overrides=topic_timeout_overrides,
+        matlab_max_attempts=args.matlab_max_attempts,
     )
     contract_topics = None if full_suite else set(selected_topic_stems)
     report["parity_contract"] = _evaluate_parity_contract(report["helpfile_similarity"]["rows"], topics_filter=contract_topics)

Original file line number	Diff line number	Diff line change
`@@ -8,3 +8,4 @@ Standalone Python port of nSTAT with MATLAB-help topic coverage and executable n`
`8`	`8`
`9`	`9`	`api`
`10`	`10`	`help_topics`
	`11`	`+ parity_runbook`