Merge remote-tracking branch 'origin/master' into HEAD

oharboe · oharboe · commit 880a34faa830 · 2024-10-11T07:11:08.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -95,3 +95,6 @@ mainREADME.md
 build
 .scala-build/
 .bsp/
+
+# autotuner artifacts
+metadata-base-at.json
diff --git a/docs/user/FlowVariables.md b/docs/user/FlowVariables.md
diff --git a/flow/Makefile b/flow/Makefile
@@ -262,7 +262,7 @@ YOSYS_FLAGS += -v 3
 
 #-------------------------------------------------------------------------------
 # setup all commands used within this flow
-export TIME_BIN   ?= /usr/bin/time
+export TIME_BIN   ?= env time
 TIME_CMD = $(TIME_BIN) -f 'Elapsed time: %E[h:]min:sec. CPU time: user %U sys %S (%P). Peak memory: %MKB.'
 TIME_TEST = $(shell $(TIME_CMD) echo foo 2>/dev/null)
 ifeq (,$(strip $(TIME_TEST)))
@@ -498,8 +498,7 @@ do-yosys:
 .PHONY: do-yosys-canonicalize
 do-yosys-canonicalize: yosys-dependencies
 	mkdir -p $(RESULTS_DIR) $(LOG_DIR) $(REPORTS_DIR) $(OBJECTS_DIR)
-	# NOTE! YOSYS_FLAGS is omitted here because "-v 3" silences helpful error messages
-	($(TIME_CMD) $(YOSYS_EXE) -c $(SCRIPTS_DIR)/synth_canonicalize.tcl) 2>&1 | tee $(abspath $(LOG_DIR)/1_1_yosys_canonicalize.log)
+	($(TIME_CMD) $(YOSYS_EXE) $(YOSYS_FLAGS) -c $(SCRIPTS_DIR)/synth_canonicalize.tcl) 2>&1 | tee $(abspath $(LOG_DIR)/1_1_yosys_canonicalize.log)
 
 $(RESULTS_DIR)/1_synth.rtlil: $(YOSYS_DEPENDENCIES)
 	$(UNSET_AND_MAKE) do-yosys-canonicalize
diff --git a/flow/designs/asap7/gcd/autotuner.json b/flow/designs/asap7/gcd/autotuner.json
@@ -44,7 +44,7 @@
         "type": "float",
         "minmax": [
             0.0,
-            0.99
+            0.1
         ],
         "step": 0
     },
@@ -64,5 +64,5 @@
         ],
         "step": 1
     },
-    "_FR_FILE_PATH": ""
+    "_FR_FILE_PATH": "fastroute.tcl"
 }
diff --git a/flow/designs/sky130hd/gcd/autotuner.json b/flow/designs/sky130hd/gcd/autotuner.json
@@ -65,14 +65,6 @@
         ],
         "step": 0
     },
-    "_PINS_DISTANCE": {
-        "type": "int",
-        "minmax": [
-            1,
-            1
-        ],
-        "step": 1
-    },
     "CTS_CLUSTER_SIZE": {
         "type": "int",
         "minmax": [
@@ -99,4 +91,3 @@
         "step": 0
     }
 }
-
diff --git a/flow/scripts/defaults.py b/flow/scripts/defaults.py
@@ -10,6 +10,6 @@
     data = yaml.safe_load(file)
 
 for key, value in data.items():
-    if value.get("value", None) is None:
+    if value.get("default", None) is None:
         continue
-    print(f'{key}?={value["value"]}')
+    print(f'{key}?={value["default"]}')
diff --git a/flow/scripts/generate-variables-docs.py b/flow/scripts/generate-variables-docs.py
@@ -25,12 +25,16 @@
 markdown_table = ""
 
 markdown_table += "## Variables in alphabetic order\n\n"
-table_header = "| Variable | Description |\n| --- | --- |\n"
+table_header = "| Variable | Description | Default |\n| --- | --- | --- |\n"
 table_rows = ""
 for key in sorted(data):
     value = data[key]
     description = value.get("description", "").replace("\n", " ").strip()
-    table_rows += f'| <a name="{key}"></a>{key} | {description} |\n'
+    table_rows += (
+        f'| <a name="{key}"></a>{key}'
+        + f"| {description}"
+        + f'| {value.get("default", "")} |\n'
+    )
 
 markdown_table += table_header + table_rows
 
diff --git a/flow/scripts/variables.yaml b/flow/scripts/variables.yaml
@@ -24,7 +24,7 @@ GENERATE_ARTIFACTS_ON_FAILURE:
     Considering that global and detailed route can run for a long time and
     use a lot of memory, this allows inspecting results on a laptop for
     a build that ran on a server.
-  value: 0
+  default: 0
 
 TNS_END_PERCENT:
   description: >
@@ -33,13 +33,13 @@ TNS_END_PERCENT:
 
     Specifies how many percent of violating paths to fix [0-100].
     Worst path will always be fixed.
-  value: 100
+  default: 100
   stages:
     - cts
     - floorplan
     - grt
 ROUTING_LAYER_ADJUSTMENT:
-  value: 0.5
+  default: 0.5
   description: Default routing layer adjustment
   stages:
     - place
@@ -49,21 +49,21 @@ ROUTING_LAYER_ADJUSTMENT:
 RECOVER_POWER:
   description: >
     Specifies how many percent of paths with positive slacks can be slowed for power savings [0-100].
-  value: 0
+  default: 0
 SKIP_INCREMENTAL_REPAIR:
-  value: 0
+  default: 0
 DETAILED_METRICS:
   description: >
     If set, then calls report_metrics prior to repair operations in the CTS
     and global route stages 
-  value: 0
+  default: 0
   stages:
     - cts
     - grt
 EQUIVALENCE_CHECK:
   description: >
     Enable running equivalence checks to verify logical correctness of repair_timing.
-  value: 0
+  default: 0
   stages:
     - cts
 CORE_UTILIZATION:
diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh
@@ -110,6 +110,14 @@ if [ $RUN_AUTOTUNER -eq 1 ]; then
 
   echo "Running Autotuner smoke sweep test"
   python3 -m unittest tools.AutoTuner.test.smoke_test_sweep.${PLATFORM}SweepSmokeTest.test_sweep
+
+  echo "Running Autotuner ref file test (only once)"
+  if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN" == "gcd" ]; then
+    python3 -m unittest tools.AutoTuner.test.ref_file_check.RefFileCheck.test_files
+  fi
+
+  echo "Running Autotuner smoke algorithm & evaluation test"
+  python3 -m unittest tools.AutoTuner.test.smoke_test_algo_eval.${PLATFORM}AlgoEvalSmokeTest.test_algo_eval
 fi
 
 exit $ret
diff --git a/flow/util/utils.mk b/flow/util/utils.mk
@@ -34,6 +34,13 @@ update_rules:
 update_rules_force:
 	$(UTILS_DIR)/genRuleFile.py $(DESIGN_DIR) --variant $(FLOW_VARIANT) --update
 
+.PHONY: update_metadata_autotuner
+update_metadata_autotuner:
+	@$(UTILS_DIR)/genMetrics.py -d $(DESIGN_NICKNAME) \
+		-p $(PLATFORM) \
+		-v $(FLOW_VARIANT) \
+		-o $(DESIGN_DIR)/metadata-$(FLOW_VARIANT)-at.json -x
+
 #-------------------------------------------------------------------------------
 
 .PHONY: write_net_rc
diff --git a/tools/AutoTuner/requirements.txt b/tools/AutoTuner/requirements.txt
@@ -1,7 +1,6 @@
 ray[default,tune]==2.9.3
 ax-platform>=0.3.3,<=0.3.7
 hyperopt==0.2.7
-nevergrad==1.0.2
 optuna==3.6.0
 pandas>=2.0,<=2.2.1
 bayesian-optimization==1.4.0
diff --git a/tools/AutoTuner/src/autotuner/distributed.py b/tools/AutoTuner/src/autotuner/distributed.py
@@ -31,10 +31,12 @@
 import sys
 import glob
 import subprocess
+import random
 from datetime import datetime
 from multiprocessing import cpu_count
 from subprocess import run
 from itertools import product
+from collections import namedtuple
 from uuid import uuid4 as uuid
 
 import numpy as np
@@ -47,12 +49,9 @@
 from ray.tune.search.ax import AxSearch
 from ray.tune.search.basic_variant import BasicVariantGenerator
 from ray.tune.search.hyperopt import HyperOptSearch
-
-# from ray.tune.search.nevergrad import NevergradSearch
 from ray.tune.search.optuna import OptunaSearch
 from ray.util.queue import Queue
 
-# import nevergrad as ng
 from ax.service.ax_client import AxClient
 
 DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
@@ -188,6 +187,9 @@ def percent(x_1, x_2):
     def evaluate(self, metrics):
         error = "ERR" in metrics.values() or "ERR" in reference.values()
         not_found = "N/A" in metrics.values() or "N/A" in reference.values()
+        print("Metrics", metrics.values())
+        print("Reference", reference.values())
+        print(error, not_found)
         if error or not_found:
             return ERROR_METRIC
         ppa = self.get_ppa(metrics)
@@ -207,6 +209,10 @@ def read_config(file_name):
     """
 
     def read(path):
+        # if file path does not exist, return empty string
+        print(os.path.abspath(path))
+        if not os.path.isfile(os.path.abspath(path)):
+            return ""
         with open(os.path.abspath(path), "r") as file:
             ret = file.read()
         return ret
@@ -221,24 +227,19 @@ def apply_condition(config, data):
         if args.algorithm != "random":
             return config
         dp_pad_min = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["minmax"][0]
-        # dp_pad_max = data['CELL_PAD_IN_SITES_DETAIL_PLACEMENT']['minmax'][1]
         dp_pad_step = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["step"]
         if dp_pad_step == 1:
             config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from(
-                lambda spec: tune.randint(
+                lambda spec: np.random.randint(
                     dp_pad_min, spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1
                 )
             )
         if dp_pad_step > 1:
             config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from(
-                lambda spec: tune.choice(
-                    np.ndarray.tolist(
-                        np.arange(
-                            dp_pad_min,
-                            spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1,
-                            dp_pad_step,
-                        )
-                    )
+                lambda spec: random.randrange(
+                    dp_pad_min,
+                    spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1,
+                    dp_pad_step,
                 )
             )
         return config
@@ -248,13 +249,8 @@ def read_tune(this):
         if min_ == max_:
             # Returning a choice of a single element allow pbt algorithm to
             # work. pbt does not accept single values as tunable.
-            return tune.choice([min_])
+            return tune.choice([min_, max_])
         if this["type"] == "int":
-            if min_ == 0 and args.algorithm == "nevergrad":
-                print(
-                    "[WARNING TUN-0011] NevergradSearch may not work "
-                    "with lower bound value 0."
-                )
             if this["step"] == 1:
                 return tune.randint(min_, max_)
             return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"])))
@@ -265,7 +261,12 @@ def read_tune(this):
         return None
 
     def read_tune_ax(name, this):
+        """
+        Ax format: https://ax.dev/versions/0.3.7/api/service.html
+        """
         dict_ = dict(name=name)
+        if "minmax" not in this:
+            return None
         min_, max_ = this["minmax"]
         if min_ == max_:
             dict_["type"] = "fixed"
@@ -292,6 +293,21 @@ def read_tune_ax(name, this):
                 dict_["value_type"] = "float"
         return dict_
 
+    def read_tune_pbt(name, this):
+        """
+        PBT format: https://docs.ray.io/en/releases-2.9.3/tune/examples/pbt_guide.html
+        Note that PBT does not support step values.
+        """
+        if "minmax" not in this:
+            return None
+        min_, max_ = this["minmax"]
+        if min_ == max_:
+            return ray.tune.choice([min_, max_])
+        if this["type"] == "int":
+            return ray.tune.randint(min_, max_)
+        if this["type"] == "float":
+            return ray.tune.uniform(min_, max_)
+
     # Check file exists and whether it is a valid JSON file.
     assert os.path.isfile(file_name), f"File {file_name} not found."
     try:
@@ -319,13 +335,25 @@ def read_tune_ax(name, this):
             fr_file = read(f"{os.path.dirname(file_name)}/{value}")
             continue
         if not isinstance(value, dict):
-            config[key] = value
+            # To take care of empty values like _FR_FILE_PATH
+            if args.mode == "tune" and args.algorithm == "ax":
+                param_dict = read_tune_ax(key, value)
+                if param_dict:
+                    config.append(param_dict)
+            elif args.mode == "tune" and args.algorithm == "pbt":
+                param_dict = read_tune_pbt(key, value)
+                if param_dict:
+                    config[key] = param_dict
+            else:
+                config[key] = value
         elif args.mode == "sweep":
             config[key] = read_sweep(value)
-        elif args.mode == "tune" and args.algorithm != "ax":
-            config[key] = read_tune(value)
         elif args.mode == "tune" and args.algorithm == "ax":
             config.append(read_tune_ax(key, value))
+        elif args.mode == "tune" and args.algorithm == "pbt":
+            config[key] = read_tune_pbt(key, value)
+        elif args.mode == "tune":
+            config[key] = read_tune(value)
     if args.mode == "tune":
         config = apply_condition(config, data)
     return config, sdc_file, fr_file
@@ -418,7 +446,10 @@ def write_sdc(variables, path):
     """
     Create a SDC file with parameters for current tuning iteration.
     """
-    # TODO: handle case where the reference file does not exist
+    # Handle case where the reference file does not exist
+    if SDC_ORIGINAL == "":
+        print("[ERROR TUN-0020] No SDC reference file provided.")
+        sys.exit(1)
     new_file = SDC_ORIGINAL
     for key, value in variables.items():
         if key == "CLK_PERIOD":
@@ -457,7 +488,10 @@ def write_fast_route(variables, path):
     """
     Create a FastRoute Tcl file with parameters for current tuning iteration.
     """
-    # TODO: handle case where the reference file does not exist
+    # Handle case where the reference file does not exist (asap7 doesn't have reference)
+    if FR_ORIGINAL == "" and args.platform != "asap7":
+        print("[ERROR TUN-0021] No FastRoute Tcl reference file provided.")
+        sys.exit(1)
     layer_cmd = "set_global_routing_layer_adjustment"
     new_file = FR_ORIGINAL
     for key, value in variables.items():
@@ -724,7 +758,7 @@ def parse_arguments():
     tune_parser.add_argument(
         "--algorithm",
         type=str,
-        choices=["hyperopt", "ax", "nevergrad", "optuna", "pbt", "random"],
+        choices=["hyperopt", "ax", "optuna", "pbt", "random"],
         default="hyperopt",
         help="Search algorithm to use for Autotuning.",
     )
@@ -840,18 +874,13 @@ def set_algorithm(experiment_name, config):
         algorithm = HyperOptSearch(points_to_evaluate=best_params)
     elif args.algorithm == "ax":
         ax_client = AxClient(enforce_sequential_optimization=False)
+        AxClientMetric = namedtuple("AxClientMetric", "minimize")
         ax_client.create_experiment(
             name=experiment_name,
             parameters=config,
-            objective_name=METRIC,
-            minimize=True,
+            objectives={METRIC: AxClientMetric(minimize=True)},
         )
         algorithm = AxSearch(ax_client=ax_client, points_to_evaluate=best_params)
-    elif args.algorithm == "nevergrad":
-        algorithm = NevergradSearch(
-            points_to_evaluate=best_params,
-            optimizer=ng.optimizers.registry["PortfolioDiscreteOnePlusOne"],
-        )
     elif args.algorithm == "optuna":
         algorithm = OptunaSearch(points_to_evaluate=best_params, seed=args.seed)
     elif args.algorithm == "pbt":
diff --git a/tools/AutoTuner/test/files/no_fr_ref.json b/tools/AutoTuner/test/files/no_fr_ref.json
diff --git a/tools/AutoTuner/test/files/no_sdc_ref.json b/tools/AutoTuner/test/files/no_sdc_ref.json
diff --git a/tools/AutoTuner/test/ref_file_check.py b/tools/AutoTuner/test/ref_file_check.py
diff --git a/tools/AutoTuner/test/smoke_test_algo_eval.py b/tools/AutoTuner/test/smoke_test_algo_eval.py
