Merge pull request #2870 from jeffng-or/at-enable-code-coverage2

Enabled code coverage for AutoTuner smoke tests

Author: vvbandeira

tools/AutoTuner/test/autotuner_test_utils.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import os
+
+
+class AutoTunerTestUtils:
+    @staticmethod
+    def get_exec_cmd():
+        """
+        Returns the execution command based on whether this is a coverage run or
+        not.
+
+        Note that you need to run coverage combine after the runs complete to
+        get the coverage of the parent plus the child invocations
+        """
+
+        if "COVERAGE_RUN" in os.environ:
+            exec = "coverage run --parallel-mode --omit=*/site-packages/*,*/dist-packages/*"
+        else:  # pragma: no cover
+            exec = "python3"
+        return exec + " -m autotuner.distributed"
+
+
+if __name__ == "__main__":  # pragma: no cover
+    print(AutoTunerTestUtils.get_exec_cmd())

tools/AutoTuner/test/ref_file_check.py

@@ -1,6 +1,7 @@
 import unittest
 import subprocess
 import os
+from .autotuner_test_utils import AutoTunerTestUtils
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 src_dir = os.path.join(cur_dir, "../src")
@@ -18,8 +19,9 @@ def setUp(self):
             "../../test/files/no_sdc_ref.json",
             "../../test/files/no_fr_ref.json",
         ]
+        self.exec = AutoTunerTestUtils.get_exec_cmd()
         self.commands = [
-            f"python3 -m autotuner.distributed"
+            f"{self.exec}"
             f" --design {self.design}"
             f" --platform {self.platform}"
             f" --config {c}"

tools/AutoTuner/test/resume_check.py

@@ -2,6 +2,7 @@
 import subprocess
 import os
 import time
+from .autotuner_test_utils import AutoTunerTestUtils
 
 from contextlib import contextmanager
 
@@ -45,8 +46,9 @@ def setUp(self):
         # Cast to 1 decimal place
         res_per_trial = float("{:.1f}".format(self.num_cpus / self.samples))
         options = ["", "--resume"]
+        self.exec = AutoTunerTestUtils.get_exec_cmd()
         self.commands = [
-            f"python3 -m autotuner.distributed"
+            f"{self.exec}"
             f" --design {self.design}"
             f" --platform {self.platform}"
             f" --config {self.config}"

tools/AutoTuner/test/smoke_test_algo_eval.py

@@ -1,6 +1,7 @@
 import unittest
 import subprocess
 import os
+from .autotuner_test_utils import AutoTunerTestUtils
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 orfs_dir = os.path.join(cur_dir, "../../../flow")
@@ -19,8 +20,9 @@ def setUp(self):
         _algo = ["hyperopt", "ax", "optuna", "pbt", "random"]
         _eval = ["default", "ppa-improv"]
         self.matrix = [(a, e) for a in _algo for e in _eval]
+        self.exec = AutoTunerTestUtils.get_exec_cmd()
         self.commands = [
-            f"python3 -m autotuner.distributed"
+            f"{self.exec}"
             f" --design {self.design}"
             f" --platform {self.platform}"
             f" --experiment {self.experiment}"

tools/AutoTuner/test/smoke_test_sample_iteration.py

@@ -1,6 +1,7 @@
 import unittest
 import subprocess
 import os
+from .autotuner_test_utils import AutoTunerTestUtils
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -16,8 +17,9 @@ def setUp(self):
         )
         self.experiment = f"smoke-test-sample-iteration-{self.platform}"
         self.matrix = [(5, 1), (1, 5), (2, 2), (1, 1)]
+        self.exec = AutoTunerTestUtils.get_exec_cmd()
         self.commands = [
-            f"python3 -m autotuner.distributed"
+            f"{self.exec}"
             f" --design {self.design}"
             f" --platform {self.platform}"
             f" --experiment {self.experiment}"

tools/AutoTuner/test/smoke_test_sweep.py

@@ -2,6 +2,7 @@
 import subprocess
 import os
 import json
+from .autotuner_test_utils import AutoTunerTestUtils
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -30,8 +31,9 @@ def setUp(self):
         core = os.cpu_count()
         self.jobs = 4 if core >= 4 else core
         self.experiment = f"smoke-test-sweep-{self.platform}"
+        self.exec = AutoTunerTestUtils.get_exec_cmd()
         self.command = (
-            "python3 -m autotuner.distributed"
+            f"{self.exec}"
             f" --design {self.design}"
             f" --platform {self.platform}"
             f" --experiment {self.experiment}"

tools/AutoTuner/test/smoke_test_tune.py

@@ -1,6 +1,7 @@
 import unittest
 import subprocess
 import os
+from .autotuner_test_utils import AutoTunerTestUtils
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -15,8 +16,9 @@ def setUp(self):
             f"../../../flow/designs/{self.platform}/{self.design}/autotuner.json",
         )
         self.experiment = f"smoke-test-tune-{self.platform}"
+        self.exec = AutoTunerTestUtils.get_exec_cmd()
         self.command = (
-            "python3 -m autotuner.distributed"
+            f"{self.exec}"
             f" --design {self.design}"
             f" --platform {self.platform}"
             f" --experiment {self.experiment}"