smoke tests accept code 16: failure due to non-valid params - this is a failure due to params.

Signed-off-by: Jack Luar <[email protected]>

Author: luarss

tools/AutoTuner/src/autotuner/distributed.py

@@ -633,7 +633,7 @@ def main():
         # if all runs have failed
         if analysis.best_result[METRIC] == ERROR_METRIC:
             print("[ERROR TUN-0016] No successful runs found.")
-            sys.exit(1)
+            sys.exit(16)
     elif args.mode == "sweep":
         sweep()
 

tools/AutoTuner/test/autotuner_test_utils.py

@@ -35,6 +35,9 @@
 
 import os
 
+# Accepted RC: success and failed with no valid results.
+accepted_rc = [0, 16]
+
 
 class AutoTunerTestUtils:
     @staticmethod

tools/AutoTuner/test/resume_check.py

@@ -36,7 +36,7 @@
 import subprocess
 import os
 import time
-from .autotuner_test_utils import AutoTunerTestUtils
+from .autotuner_test_utils import AutoTunerTestUtils, accepted_rc
 
 from contextlib import contextmanager
 
@@ -108,7 +108,7 @@ def test_tune_resume(self):
             proc = subprocess.run("ray status", shell=True)
             no_nodes = proc.returncode != 0
             proc = subprocess.run("ray stop", shell=True)
-            successful = proc.returncode == 0
+            successful = proc.returncode in accepted_rc
 
             if no_nodes and successful:
                 break
@@ -117,7 +117,7 @@ def test_tune_resume(self):
         # Run the second config to completion
         print("Running the second config")
         proc = subprocess.run(self.commands[1], shell=True)
-        successful = proc.returncode == 0
+        successful = proc.returncode in accepted_rc
         self.assertTrue(successful)
 
 

tools/AutoTuner/test/smoke_test_algo_eval.py

@@ -35,7 +35,7 @@
 import unittest
 import subprocess
 import os
-from .autotuner_test_utils import AutoTunerTestUtils
+from .autotuner_test_utils import AutoTunerTestUtils, accepted_rc
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 orfs_dir = os.path.join(cur_dir, "../../../flow")
@@ -75,7 +75,7 @@ def make_base(self):
         ]
         for command in commands:
             out = subprocess.run(command, shell=True, check=True)
-            self.assertTrue(out.returncode == 0)
+            self.assertTrue(out.returncode in accepted_rc)
 
     def test_algo_eval(self):
         if not (self.platform and self.design):
@@ -85,7 +85,7 @@ def test_algo_eval(self):
         for command in self.commands:
             print(command)
             out = subprocess.run(command, shell=True, check=True)
-            successful = out.returncode == 0
+            successful = out.returncode in accepted_rc
             self.assertTrue(successful)
 
 

tools/AutoTuner/test/smoke_test_sample_iteration.py

@@ -35,7 +35,7 @@
 import unittest
 import subprocess
 import os
-from .autotuner_test_utils import AutoTunerTestUtils
+from .autotuner_test_utils import AutoTunerTestUtils, accepted_rc
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -68,7 +68,7 @@ def test_sample_iteration(self):
         for command in self.commands:
             print(command)
             out = subprocess.run(command, shell=True, check=True)
-            successful = out.returncode == 0
+            successful = out.returncode in accepted_rc
             self.assertTrue(successful)
 
 

tools/AutoTuner/test/smoke_test_sweep.py

@@ -36,7 +36,7 @@
 import subprocess
 import os
 import json
-from .autotuner_test_utils import AutoTunerTestUtils
+from .autotuner_test_utils import AutoTunerTestUtils, accepted_rc
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -80,7 +80,7 @@ def test_sweep(self):
         if not (self.platform and self.design):
             raise unittest.SkipTest("Platform and design have to be defined")
         out = subprocess.run(self.command, shell=True, check=True)
-        successful = out.returncode == 0
+        successful = out.returncode in accepted_rc
         self.assertTrue(successful)
 
 

tools/AutoTuner/test/smoke_test_tune.py

@@ -35,7 +35,7 @@
 import unittest
 import subprocess
 import os
-from .autotuner_test_utils import AutoTunerTestUtils
+from .autotuner_test_utils import AutoTunerTestUtils, accepted_rc
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -64,7 +64,7 @@ def test_tune(self):
         if not (self.platform and self.design):
             raise unittest.SkipTest("Platform and design have to be defined")
         out = subprocess.run(self.command, shell=True, check=True)
-        successful = out.returncode == 0
+        successful = out.returncode in accepted_rc
         self.assertTrue(successful)