ver: Add test iteration variable

Signed-off-by: Wiktoria Kuna <[email protected]>

Author: wkkuna

tools/nox_utils/src/nox_utils.py

@@ -87,27 +87,29 @@ class VerificationTest:
     Useful to manage files produced by Cocotb+Verilator in I3C_ROOT_DIR/verification/block
     """
 
-    def __init__(self, blockName: str, blockPath: str, testName: str, coverage: str | None):
+    def __init__(self, blockName: str, blockPath: str, testName: str, coverage: str | None, pfx=""):
         self.blockName = blockName
         self.blockPath = blockPath
         self.testName = testName
         self.coverage = coverage
+        self.pfx = pfx
         self.testPath = os.path.join(blockPath, blockName)
+        self.sim_build = "sim_build" if coverage is None else f"sim_build-{self.testName}-{coverage}"
 
         # Convert NoneType to empty string
         coverage = "" if coverage is None else str(coverage)
 
         # Defaults from verilator
         defaultNameVCD = "dump.vcd"
         defaultNameCoverage = "coverage.dat"
-        defaultTestNameLog = f"{testName}.log"
-        defaultNameVDB = f"sim_build-{testName}-{coverage}/simv.vdb"
-
-        testNameVCD = f"{testName}.vcd"
-        testNameXML = f"{testName}.xml"
-        testCoverageName = f"{testName}_{coverage}.dat"
-        testNameLog = f"{testName}_{coverage}.log"
-        testNameVDB = f"{testName}.vdb"
+        defaultTestNameLog = f"{self.testName}{pfx}.log"
+        defaultNameVDB = f"{self.sim_build}/simv.vdb"
+
+        testNameVCD = f"{self.testName}{pfx}.vcd"
+        testNameXML = f"{self.testName}{pfx}.xml"
+        testCoverageName = f"{self.testName}{pfx}_{coverage}.dat"
+        testNameLog = f"{self.testName}{pfx}_{coverage}.log"
+        testNameVDB = f"{self.testName}{pfx}.vdb"
 
         self.filenames = {
             "vcd_default": defaultNameVCD,

verification/cocotb/noxfile.py

@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 import random
 import time
+import shutil
 
 import nox
 from nox_utils import VerificationTest, isCocotbSimFailure, nox_config
@@ -23,55 +23,63 @@
 else:
     coverage_types = None
 
+
 def _verify(session, test_group, test_type, test_name, coverage=None, simulator=None):
     # session.install("-r", pip_requirements_path)
-    test = VerificationTest(test_group, test_type, test_name, coverage)
-
-    # Translate session options to plusargs
-    plusargs = list(session.posargs)
-
-    # Randomize seed for initialization of undefined signals in the simulation
-    random.seed(time.time_ns())
-    seed = random.randint(1, 10000)
-
-    with open(test.paths["log_default"], "w") as test_log:
-
-        args = [
-            "make",
-            "-C",
-            test.testPath,
-            "all",
-            "MODULE=" + test_name,
-            "COCOTB_RESULTS_FILE=" + test.filenames["xml"],
-        ]
-        if simulator == "verilator":
-            plusargs.extend(
-                [
-                    "+verilator+rand+reset+2",
-                    f"+verilator+seed+{seed}",
-                ]
+
+    test_iterations = int(os.getenv("TEST_ITERATIONS", 1))
+    for i in range(test_iterations):
+        pfx = "" if test_iterations == 1 else f"_{i}"
+        test = VerificationTest(test_group, test_type, test_name, coverage, pfx)
+        # Translate session options to plusargs
+        plusargs = list(session.posargs)
+
+        # Randomize seed for initialization of undefined signals in the simulation
+        random.seed(time.time_ns())
+        seed = random.randint(1, 10000)
+
+        with open(test.paths["log_default"], "w") as test_log:
+            # Remove simulation build artifacts
+            # When collecting coverage and renaming `vdb` database
+            # the following simulations will fail due to non-existent database
+            if simulator == "vcs" and i > 0:
+                shutil.rmtree(os.path.join(test.testPath, test.sim_build))
+
+            args = [
+                "make",
+                "-C",
+                test.testPath,
+                "all",
+                "MODULE=" + test_name,
+                "COCOTB_RESULTS_FILE=" + test.filenames["xml"],
+            ]
+            if simulator == "verilator":
+                plusargs.extend(
+                    [
+                        "+verilator+rand+reset+2",
+                        f"+verilator+seed+{seed}",
+                    ]
+                )
+            if coverage:
+                args.append("COVERAGE_TYPE=" + coverage)
+
+            if simulator:
+                args.append("SIM=" + simulator)
+
+            args.append("PLUSARGS=" + " ".join(plusargs))
+
+            session.run(
+                *args,
+                external=True,
+                stdout=test_log,
+                stderr=test_log,
             )
-        if coverage:
-            args.append("COVERAGE_TYPE=" + coverage)
-
-        if simulator:
-            args.append("SIM=" + simulator)
-
-        args.append("PLUSARGS=" + " ".join(plusargs))
-
-        session.run(
-            *args,
-            external=True,
-            stdout=test_log,
-            stderr=test_log,
-        )
-    # Prevent coverage.dat and test log from being overwritten
-    test.rename_defaults(coverage, simulator)
-
-    # Add check from results.xml to notify nox that test failed
-    isTBFailure = isCocotbSimFailure(resultsFile=test.paths["xml"])
-    if isTBFailure:
-        raise Exception("SimFailure: cocotb failed. See test logs for more information.")
+        # Prevent coverage.dat and test log from being overwritten
+        test.rename_defaults(coverage, simulator)
+
+        # Add check from results.xml to notify nox that test failed
+        if isCocotbSimFailure(resultsFile=test.paths["xml"]):
+            raise Exception("SimFailure: cocotb failed. See test logs for more information.")
 
 
 def verify_block(session, test_group, test_name, coverage=None, simulator=None):