tools: Add sim_repeater utility

Signed-off-by: Jan Malek <[email protected]>

Author: jan-malek

tools/nox_utils/src/nox_utils.py

@@ -4,6 +4,7 @@
 import os
 import re
 from xml.etree import ElementTree
+from pathlib import Path
 
 """
 Common functions and utilities for noxfile.py
@@ -20,6 +21,10 @@ def nox_config(nox):
     return nox
 
 
+def sim_repeater_path():
+    return str((Path(__file__).parent / ".." / ".." / "sim_repeater.sh").resolve())
+
+
 def setupLogger(verbose=False, filename="setup_logger.log"):
     logger = logging.getLogger()
     logHandler = logging.FileHandler(filename=filename, mode="w", encoding="utf-8")

tools/sim_repeater.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+SIM_TRIES="${SIM_TRIES:-3}"
+RETRY_COOLDOWN=10
+TEMP_LOGS="$(mktemp -d)"
+
+function cleanup {
+  rm -rf "${TEMP_LOGS}"
+}
+
+trap cleanup EXIT
+
+# Needed to propagate the exit code through the pipe to `tee`
+set -o pipefail
+
+for (( i=1; i<="${SIM_TRIES}"; i++ )); do
+    echo "$@"
+    "$@" 2>&1 | tee "${TEMP_LOGS}/${i}"
+    RESULT=$?
+
+    if [ -n "${SIM_RETRY_CONDITION}" ] && [ $RESULT != 0 ]; then
+        if grep -E "${SIM_RETRY_CONDITION}" "${TEMP_LOGS}/${i}" &> /dev/null; then
+            echo "Retry condition encountered. Retrying in ${RETRY_COOLDOWN}s"
+            echo ''
+            sleep "${RETRY_COOLDOWN}"
+            continue
+        fi
+    fi
+
+    exit $RESULT
+done
+
+echo "Limit of ${SIM_TRIES} tries exceeded. Exiting"
+exit 1

verification/cocotb/noxfile.py

@@ -5,7 +5,7 @@
 import shutil
 
 import nox
-from nox_utils import VerificationTest, isCocotbSimFailure, nox_config
+from nox_utils import VerificationTest, isCocotbSimFailure, nox_config, sim_repeater_path
 
 # Common nox configuration
 nox = nox_config(nox)
@@ -46,13 +46,15 @@ def _verify(session, test_group, test_type, test_name, coverage=None, simulator=
                 shutil.rmtree(os.path.join(test.testPath, test.sim_build))
 
             args = [
+                sim_repeater_path(),
                 "make",
                 "-C",
                 test.testPath,
                 "all",
                 "MODULE=" + test_name,
                 "COCOTB_RESULTS_FILE=" + test.filenames["xml"],
             ]
+
             if simulator == "verilator":
                 plusargs.extend(
                     [

verification/uvm_i3c/noxfile.py

@@ -4,7 +4,7 @@
 import os
 import nox
 
-from nox_utils import nox_config, isUVMSimFailure, create_test_id
+from nox_utils import nox_config, isUVMSimFailure, create_test_id, sim_repeater_path
 
 nox = nox_config(nox)
 
@@ -42,7 +42,7 @@ def verify_uvm(
     root_dir = setup_root_dir()
 
     make_targets = ["config", simulator]
-    args = ["make", "-C", root_dir, *make_targets, f"UVM_TB_FILES={tb_files}"]
+    args = [sim_repeater_path(), "make", "-C", root_dir, *make_targets, f"UVM_TB_FILES={tb_files}"]
 
     if uvm_testname != "":
         args.append(f"UVM_TESTNAME={uvm_testname}")