[Autotuner] CI Smoke Test - Algorithm & Evaluation mode (#2052)

* lock in scaffold for tests
* Fix bugs for random search
* Remove nevergrad code and dependencies
* remove pins distance

Signed-off-by: Jack Luar <[email protected]>

Author: luarss

.gitignore

@@ -95,3 +95,6 @@ mainREADME.md
 build
 .scala-build/
 .bsp/
+
+# autotuner artifacts
+metadata-base-at.json

flow/designs/asap7/gcd/autotuner.json

@@ -44,7 +44,7 @@
         "type": "float",
         "minmax": [
             0.0,
-            0.99
+            0.1
         ],
         "step": 0
     },

flow/designs/sky130hd/gcd/autotuner.json

@@ -65,14 +65,6 @@
         ],
         "step": 0
     },
-    "_PINS_DISTANCE": {
-        "type": "int",
-        "minmax": [
-            1,
-            1
-        ],
-        "step": 1
-    },
     "CTS_CLUSTER_SIZE": {
         "type": "int",
         "minmax": [
@@ -99,4 +91,3 @@
         "step": 0
     }
 }
-

flow/test/test_helper.sh

@@ -110,6 +110,9 @@ if [ $RUN_AUTOTUNER -eq 1 ]; then
 
   echo "Running Autotuner smoke sweep test"
   python3 -m unittest tools.AutoTuner.test.smoke_test_sweep.${PLATFORM}SweepSmokeTest.test_sweep
+
+  echo "Running Autotuner smoke algorithm & evaluation test"
+  python3 -m unittest tools.AutoTuner.test.smoke_test_algo_eval.${PLATFORM}AlgoEvalSmokeTest.test_algo_eval
 fi
 
 exit $ret

flow/util/utils.mk

@@ -34,6 +34,13 @@ update_rules:
 update_rules_force:
 	$(UTILS_DIR)/genRuleFile.py $(DESIGN_DIR) --variant $(FLOW_VARIANT) --update
 
+.PHONY: update_metadata_autotuner
+update_metadata_autotuner:
+	@$(UTILS_DIR)/genMetrics.py -d $(DESIGN_NICKNAME) \
+		-p $(PLATFORM) \
+		-v $(FLOW_VARIANT) \
+		-o $(DESIGN_DIR)/metadata-$(FLOW_VARIANT)-at.json -x
+
 #-------------------------------------------------------------------------------
 
 .PHONY: write_net_rc

tools/AutoTuner/requirements.txt

@@ -1,7 +1,6 @@
 ray[default,tune]==2.9.3
 ax-platform>=0.3.3,<=0.3.7
 hyperopt==0.2.7
-nevergrad==1.0.2
 optuna==3.6.0
 pandas>=2.0,<=2.2.1
 bayesian-optimization==1.4.0

tools/AutoTuner/src/autotuner/distributed.py

@@ -31,10 +31,12 @@
 import sys
 import glob
 import subprocess
+import random
 from datetime import datetime
 from multiprocessing import cpu_count
 from subprocess import run
 from itertools import product
+from collections import namedtuple
 from uuid import uuid4 as uuid
 
 import numpy as np
@@ -47,12 +49,9 @@
 from ray.tune.search.ax import AxSearch
 from ray.tune.search.basic_variant import BasicVariantGenerator
 from ray.tune.search.hyperopt import HyperOptSearch
-
-# from ray.tune.search.nevergrad import NevergradSearch
 from ray.tune.search.optuna import OptunaSearch
 from ray.util.queue import Queue
 
-# import nevergrad as ng
 from ax.service.ax_client import AxClient
 
 DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
@@ -188,6 +187,9 @@ def percent(x_1, x_2):
     def evaluate(self, metrics):
         error = "ERR" in metrics.values() or "ERR" in reference.values()
         not_found = "N/A" in metrics.values() or "N/A" in reference.values()
+        print("Metrics", metrics.values())
+        print("Reference", reference.values())
+        print(error, not_found)
         if error or not_found:
             return ERROR_METRIC
         ppa = self.get_ppa(metrics)
@@ -221,24 +223,19 @@ def apply_condition(config, data):
         if args.algorithm != "random":
             return config
         dp_pad_min = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["minmax"][0]
-        # dp_pad_max = data['CELL_PAD_IN_SITES_DETAIL_PLACEMENT']['minmax'][1]
         dp_pad_step = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["step"]
         if dp_pad_step == 1:
             config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from(
-                lambda spec: tune.randint(
+                lambda spec: np.random.randint(
                     dp_pad_min, spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1
                 )
             )
         if dp_pad_step > 1:
             config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from(
-                lambda spec: tune.choice(
-                    np.ndarray.tolist(
-                        np.arange(
-                            dp_pad_min,
-                            spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1,
-                            dp_pad_step,
-                        )
-                    )
+                lambda spec: random.randrange(
+                    dp_pad_min,
+                    spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1,
+                    dp_pad_step,
                 )
             )
         return config
@@ -248,13 +245,8 @@ def read_tune(this):
         if min_ == max_:
             # Returning a choice of a single element allow pbt algorithm to
             # work. pbt does not accept single values as tunable.
-            return tune.choice([min_])
+            return tune.choice([min_, max_])
         if this["type"] == "int":
-            if min_ == 0 and args.algorithm == "nevergrad":
-                print(
-                    "[WARNING TUN-0011] NevergradSearch may not work "
-                    "with lower bound value 0."
-                )
             if this["step"] == 1:
                 return tune.randint(min_, max_)
             return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"])))
@@ -265,7 +257,12 @@ def read_tune(this):
         return None
 
     def read_tune_ax(name, this):
+        """
+        Ax format: https://ax.dev/versions/0.3.7/api/service.html
+        """
         dict_ = dict(name=name)
+        if "minmax" not in this:
+            return None
         min_, max_ = this["minmax"]
         if min_ == max_:
             dict_["type"] = "fixed"
@@ -292,6 +289,21 @@ def read_tune_ax(name, this):
                 dict_["value_type"] = "float"
         return dict_
 
+    def read_tune_pbt(name, this):
+        """
+        PBT format: https://docs.ray.io/en/releases-2.9.3/tune/examples/pbt_guide.html
+        Note that PBT does not support step values.
+        """
+        if "minmax" not in this:
+            return None
+        min_, max_ = this["minmax"]
+        if min_ == max_:
+            return ray.tune.choice([min_, max_])
+        if this["type"] == "int":
+            return ray.tune.randint(min_, max_)
+        if this["type"] == "float":
+            return ray.tune.uniform(min_, max_)
+
     # Check file exists and whether it is a valid JSON file.
     assert os.path.isfile(file_name), f"File {file_name} not found."
     try:
@@ -319,13 +331,25 @@ def read_tune_ax(name, this):
             fr_file = read(f"{os.path.dirname(file_name)}/{value}")
             continue
         if not isinstance(value, dict):
-            config[key] = value
+            # To take care of empty values like _FR_FILE_PATH
+            if args.mode == "tune" and args.algorithm == "ax":
+                param_dict = read_tune_ax(key, value)
+                if param_dict:
+                    config.append(param_dict)
+            elif args.mode == "tune" and args.algorithm == "pbt":
+                param_dict = read_tune_pbt(key, value)
+                if param_dict:
+                    config[key] = param_dict
+            else:
+                config[key] = value
         elif args.mode == "sweep":
             config[key] = read_sweep(value)
-        elif args.mode == "tune" and args.algorithm != "ax":
-            config[key] = read_tune(value)
         elif args.mode == "tune" and args.algorithm == "ax":
             config.append(read_tune_ax(key, value))
+        elif args.mode == "tune" and args.algorithm == "pbt":
+            config[key] = read_tune_pbt(key, value)
+        elif args.mode == "tune":
+            config[key] = read_tune(value)
     if args.mode == "tune":
         config = apply_condition(config, data)
     return config, sdc_file, fr_file
@@ -724,7 +748,7 @@ def parse_arguments():
     tune_parser.add_argument(
         "--algorithm",
         type=str,
-        choices=["hyperopt", "ax", "nevergrad", "optuna", "pbt", "random"],
+        choices=["hyperopt", "ax", "optuna", "pbt", "random"],
         default="hyperopt",
         help="Search algorithm to use for Autotuning.",
     )
@@ -840,18 +864,13 @@ def set_algorithm(experiment_name, config):
         algorithm = HyperOptSearch(points_to_evaluate=best_params)
     elif args.algorithm == "ax":
         ax_client = AxClient(enforce_sequential_optimization=False)
+        AxClientMetric = namedtuple("AxClientMetric", "minimize")
         ax_client.create_experiment(
             name=experiment_name,
             parameters=config,
-            objective_name=METRIC,
-            minimize=True,
+            objectives={METRIC: AxClientMetric(minimize=True)},
         )
         algorithm = AxSearch(ax_client=ax_client, points_to_evaluate=best_params)
-    elif args.algorithm == "nevergrad":
-        algorithm = NevergradSearch(
-            points_to_evaluate=best_params,
-            optimizer=ng.optimizers.registry["PortfolioDiscreteOnePlusOne"],
-        )
     elif args.algorithm == "optuna":
         algorithm = OptunaSearch(points_to_evaluate=best_params, seed=args.seed)
     elif args.algorithm == "pbt":

tools/AutoTuner/test/smoke_test_algo_eval.py

@@ -0,0 +1,92 @@
+import unittest
+import subprocess
+import os
+
+cur_dir = os.path.dirname(os.path.abspath(__file__))
+src_dir = os.path.join(cur_dir, "../src/autotuner")
+orfs_dir = os.path.join(cur_dir, "../../../flow")
+os.chdir(src_dir)
+
+
+class BaseAlgoEvalSmokeTest(unittest.TestCase):
+    platform = ""
+    design = ""
+
+    def setUp(self):
+        design_path = f"../../../flow/designs/{self.platform}/{self.design}"
+        self.config = os.path.join(cur_dir, f"{design_path}/autotuner.json")
+        self.experiment = f"smoke-test-algo-eval-{self.platform}"
+        self.reference = os.path.join(cur_dir, f"{design_path}/metadata-base-at.json")
+        # note for ppa-improv, you need to also add in reference file (--reference)
+        _algo = ["hyperopt", "ax", "optuna", "pbt", "random"]
+        _eval = ["default", "ppa-improv"]
+        self.matrix = [(a, e) for a in _algo for e in _eval]
+        self.commands = [
+            f"python3 distributed.py"
+            f" --design {self.design}"
+            f" --platform {self.platform}"
+            f" --experiment {self.experiment}"
+            f" --config {self.config}"
+            f" tune --samples 1"
+            f" --algorithm {a} --eval {e}"
+            f" --reference {self.reference}"
+            for a, e in self.matrix
+        ]
+
+    def make_base(self):
+        os.chdir(orfs_dir)
+        commands = [
+            f"make DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk clean_all",
+            f"make DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk EQUIVALENCE_CHECK=0",
+            f"make DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk update_metadata_autotuner",
+        ]
+        for command in commands:
+            out = subprocess.run(command, shell=True, check=True)
+            self.assertTrue(out.returncode == 0)
+        os.chdir(src_dir)
+
+
+class ASAP7AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
+    platform = "asap7"
+    design = "gcd"
+
+    def test_algo_eval(self):
+        # Run `make` to get baseline metrics (metadata-base-ok.json)
+        self.make_base()
+        for command in self.commands:
+            print(command)
+            out = subprocess.run(command, shell=True, check=True)
+            successful = out.returncode == 0
+            self.assertTrue(successful)
+
+
+class IHPSG13G2AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
+    platform = "ihp-sg13g2"
+    design = "gcd"
+
+    def test_algo_eval(self):
+        # Run `make` to get baseline metrics (metadata-base-ok.json)
+        self.make_base()
+        for command in self.commands:
+            print(command)
+            out = subprocess.run(command, shell=True, check=True)
+            successful = out.returncode == 0
+            self.assertTrue(successful)
+
+
+class SKY130HDAlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
+    platform = "sky130hd"
+    design = "gcd"
+
+    def test_algo_eval(self):
+        # Run `make` to get baseline metrics (metadata-base-ok.json)
+        self.make_base()
+        for command in self.commands:
+            print(command)
+            out = subprocess.run(command, shell=True, check=True)
+            successful = out.returncode == 0
+            self.assertTrue(successful)
+
+
+if __name__ == "__main__":
+    unittest.main()