Sweep experiments script (#504)

* Experiment updates

* PR feedback

Author: tgerdesnv

experiments/checkpoint_experiment_data.py

@@ -15,6 +15,7 @@
 from experiments.experiment_data import ExperimentData
 from model_analyzer.state.analyzer_state_manager import AnalyzerStateManager
 from unittest.mock import MagicMock
+from copy import deepcopy
 
 
 class CheckpointExperimentData(ExperimentData):
@@ -24,18 +25,31 @@ class CheckpointExperimentData(ExperimentData):
 
     def __init__(self, config):
         super().__init__()
+        self._default_run_config = None
         self._load_checkpoint(config)
 
+    def get_default_config_dict(self):
+        ret = self._default_run_config.model_run_configs()[0].model_config(
+        ).to_dict()
+        ret = deepcopy(ret)
+        del ret["cpu_only"]
+        return ret
+
     def _load_checkpoint(self, config):
         state_manager = AnalyzerStateManager(config, MagicMock())
         state_manager.load_checkpoint(checkpoint_required=True)
 
         results = state_manager.get_state_variable('ResultManager.results')
+
         model_name = ",".join([x.model_name() for x in config.profile_models])
         model_measurements = results.get_model_measurements_dict(model_name)
         for (run_config,
              run_config_measurements) in model_measurements.values():
 
+            if run_config.model_variants_name(
+            ) == model_name + "_config_default":
+                self._default_run_config = run_config
+
             # Due to the way that data is stored in the AnalyzerStateManager, the
             # run_config only represents the model configuration used. The
             # perf_analyzer information for each measurement associated with it
@@ -46,6 +60,46 @@ def _load_checkpoint(self, config):
             for (perf_analyzer_string,
                  run_config_measurement) in run_config_measurements.items():
 
+                run_config_measurement.set_model_config_constraints(
+                    model_config_constraints=[config.constraints])
+                run_config_measurement.set_metric_weightings(
+                    metric_objectives=[config.objectives])
                 pa_key = self._make_pa_key_from_cli_string(perf_analyzer_string)
-                self._add_run_config_measurement_from_keys(
-                    ma_key, pa_key, run_config, run_config_measurement)
+
+                existing_measurement = self._get_run_config_measurement_from_keys(
+                    ma_key, pa_key, skip_warn=True)
+                if not existing_measurement or run_config_measurement > existing_measurement:
+                    self._add_run_config_measurement_from_keys(
+                        ma_key, pa_key, run_config, run_config_measurement)
+
+        if self._default_run_config is None:
+            print(f"No default config for {model_name}")
+            exit(1)
+
+        self._print_map()
+
+    def _print_map(self):
+        for i in range(0, 10):
+            row_str = ""
+            for j in range(0, 10):
+                instance_count = j + 1
+                max_batch_size = 2**i
+
+                ma_key = f"instance_count={instance_count},max_batch_size={max_batch_size}"
+
+                clamped_int = self._clamp_to_power_of_two(2 * instance_count *
+                                                          max_batch_size)
+
+                pa_key = str(clamped_int)
+
+                measurement = self._get_run_config_measurement_from_keys(
+                    ma_key, pa_key, skip_warn=True)
+                tput = 0
+                lat = 0
+                if measurement:
+                    tput = measurement.get_non_gpu_metric_value(
+                        'perf_throughput')
+                    lat = measurement.get_non_gpu_metric_value(
+                        'perf_latency_p99')
+                row_str += f"\t{tput:4.1f}:{lat:4.1f}"
+            print(row_str)

experiments/config_command_experiment.py

@@ -24,6 +24,16 @@ class ConfigCommandExperiment(ConfigCommandProfile):
 
     def _fill_config(self):
         super()._fill_config()
+        self._add_config(
+            ConfigField(
+                'exponential_inst_count',
+                field_type=ConfigPrimitive(bool),
+                flags=['--exponential-inst-count'],
+                parser_args={'action': 'store_true'},
+                default_value=False,
+                description=
+                'Whether or not the inst count dimension should be linear or exponential'
+            ))
         self._add_config(
             ConfigField('radius',
                         field_type=ConfigPrimitive(int),

experiments/evaluate_config_generator.py

@@ -18,6 +18,7 @@
 from experiment_data import ExperimentData
 from checkpoint_experiment_data import CheckpointExperimentData
 from experiment_file_writer import ExperimentFileWriter
+from unittest.mock import MagicMock, patch
 
 
 class EvaluateConfigGenerator:
@@ -35,6 +36,13 @@ def __init__(self, model_name, data_path, output_path, other_args):
         self._checkpoint_data = CheckpointExperimentData(self._config_command)
         self._profile_data = ExperimentData()
 
+        self._default_config_dict = self._checkpoint_data.get_default_config_dict(
+        )
+        p = patch(
+            'model_analyzer.config.generate.base_model_config_generator.BaseModelConfigGenerator.get_base_model_config_dict',
+            MagicMock(return_value=self._default_config_dict))
+        p.start()
+
     def execute_generator(self, generator_name):
 
         generator = GeneratorExperimentFactory.create_generator(
@@ -59,11 +67,14 @@ def _run_generator(self, cg):
         for run_config in cg.get_configs():
             run_config_measurement = self._checkpoint_data.get_run_config_measurement(
                 run_config)
-            self._profile_data.add_run_config_measurement(
-                run_config, run_config_measurement)
 
             if run_config_measurement:
+                run_config_measurement.set_metric_weightings(
+                    metric_objectives=[self._config_command.objectives])
                 run_config_measurement.set_model_config_constraints(
                     model_config_constraints=[self._config_command.constraints])
 
+            self._profile_data.add_run_config_measurement(
+                run_config, run_config_measurement)
+
             cg.set_last_results([run_config_measurement])

experiments/experiment_config_command_creator.py

@@ -12,11 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from tests.common.test_utils import convert_to_bytes
 from tests.mocks.mock_config import MockConfig
 from tests.mocks.mock_model_config import MockModelConfig
+from tests.common.test_utils import convert_to_bytes
 from model_analyzer.cli.cli import CLI
 from config_command_experiment import ConfigCommandExperiment
+import re
 
 
 class ExperimentConfigCommandCreator:
@@ -26,22 +27,35 @@ class ExperimentConfigCommandCreator:
 
     @staticmethod
     def make_config(data_path, model_name, other_args):
-        mock_model_config = MockModelConfig("")
-        mock_model_config.start()
 
-        checkpoint_dir = f"{data_path}/{model_name}"
+        ckpt = re.search('(.+)\/(\d\.ckpt)', data_path)
+        if ckpt:
+            checkpoint_dir = ckpt.group(1)
+        else:
+            checkpoint_dir = f"{data_path}/{model_name}"
 
         #yapf: disable
         args = [
             'model-analyzer', 'profile',
             '--profile-models', model_name,
             '--model-repository', data_path,
-            '--checkpoint-directory', checkpoint_dir,
-            '-f', 'path-to-config-file'
+            '--checkpoint-directory', checkpoint_dir
         ]
         args += other_args
 
-        yaml_content = convert_to_bytes("")
+        if '-f' not in args and '--config-file' not in args:
+            args += ['-f', 'path-to-config-file']
+            yaml_content = convert_to_bytes("")
+        else:
+            index = args.index('-f') if '-f' in args else args.index('--config-file')
+            yaml_file = args[index + 1]
+
+            with open(yaml_file, 'r') as f:
+                yaml_content = f.read()
+                yaml_content = convert_to_bytes(yaml_content)
+
+        mock_model_config = MockModelConfig("")
+        mock_model_config.start()
 
         mock_config = MockConfig(args, yaml_content)
         mock_config.start()

experiments/experiment_data.py

@@ -87,23 +87,27 @@ def _add_run_config_measurement_from_keys(self, ma_key, pa_key, run_config,
         curr_dict[ma_key][pa_key] = run_config_measurement
 
     def _update_best_trackers(self, run_config, run_config_measurement):
-        if not self._best_run_config_measurement or run_config_measurement.get_non_gpu_metric_value(
-                'perf_throughput'
-        ) > self._best_run_config_measurement.get_non_gpu_metric_value(
-                'perf_throughput'):
+        if run_config_measurement.is_passing_constraints() and \
+            (not self._best_run_config_measurement or (run_config_measurement > self._best_run_config_measurement)):
+
             self._best_run_config_measurement = run_config_measurement
             self._best_run_config = run_config
 
-    def _get_run_config_measurement_from_keys(self, ma_key, pa_key):
+    def _get_run_config_measurement_from_keys(self,
+                                              ma_key,
+                                              pa_key,
+                                              skip_warn=False):
         if ma_key not in self._data:
-            print(f"WARNING: Model config {ma_key} not in results")
-            self._missing_measurement_count += 1
+            if not skip_warn:
+                print(f"WARNING: Model config {ma_key} not in results")
+                self._missing_measurement_count += 1
             return None
         if pa_key not in self._data[ma_key]:
-            print(
-                f"WARNING: Model config {ma_key}, concurrency={pa_key} not in results"
-            )
-            self._missing_measurement_count += 1
+            if not skip_warn:
+                print(
+                    f"WARNING: Model config {ma_key}, concurrency={pa_key} not in results"
+                )
+                self._missing_measurement_count += 1
             return None
 
         return self._data[ma_key][pa_key]

experiments/experiment_evaluator.py

@@ -12,14 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from experiment_data import ExperimentData
+
 
 class ExperimentEvaluator:
     """ 
     Class to compare the results of a config generator execution against
     a checkpoint of raw data
     """
 
-    def __init__(self, raw_data, profile_data):
+    def __init__(self, raw_data: ExperimentData, profile_data: ExperimentData):
         self._raw_data = raw_data
         self._profile_data = profile_data
 
@@ -43,6 +45,9 @@ def print_results(self):
         print(
             f"Overall best throughput: {overall_best_measurement.get_non_gpu_metric_value('perf_throughput')}"
         )
+        print(
+            f"Overall best latency: {overall_best_measurement.get_non_gpu_metric_value('perf_latency_p99')}"
+        )
         print()
         print(
             f"Generator num measurements: {self._profile_data.get_run_config_measurement_count()}"
@@ -56,14 +61,29 @@ def print_results(self):
         print(
             f"Generator best config: {self._run_config_to_string(generator_best_run_config)}"
         )
-        print(
-            f"Generator best throughput: {generator_best_measurement.get_non_gpu_metric_value('perf_throughput')}"
-        )
+
+        if generator_best_measurement:
+            best_throughput = generator_best_measurement.get_non_gpu_metric_value(
+                'perf_throughput')
+            best_latency = generator_best_measurement.get_non_gpu_metric_value(
+                'perf_latency_p99')
+            overall_best_throughput = overall_best_measurement.get_non_gpu_metric_value(
+                'perf_throughput')
+            percentile = round(best_throughput / overall_best_throughput, 2)
+        else:
+            best_throughput = None
+            best_latency = None
+            percentile = None
+
+        print(f"Generator best throughput: {best_throughput}")
+        print(f"Generator best latency: {best_latency}")
+        print(f"Percentile: {percentile}")
         print()
 
     def _run_config_to_string(self, run_config):
-        str = "\n".join([
-            f"{x.model_config().get_config()}"
-            for x in run_config.model_run_configs()
-        ])
-        return str
+        if run_config:
+            str = "\n".join([
+                f"{x.model_config().get_config()}"
+                for x in run_config.model_run_configs()
+            ])
+            return str

experiments/generator_experiment_factory.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from model_analyzer.config.generate.brute_run_config_generator import BruteRunConfigGenerator
+from model_analyzer.config.generate.model_variant_name_manager import ModelVariantNameManager
 from model_analyzer.config.generate.quick_run_config_generator import QuickRunConfigGenerator
 from model_analyzer.config.generate.search_config import SearchConfig
 from model_analyzer.config.generate.search_dimension import SearchDimension
@@ -53,10 +54,17 @@ def create_generator(generator_name, config_command):
 
             #yapf: disable
             for i, _ in enumerate(config_command.profile_models):
-                dimensions.add_dimensions(i, [
-                    SearchDimension(f"max_batch_size", SearchDimension.DIMENSION_TYPE_EXPONENTIAL),
-                    SearchDimension(f"instance_count", SearchDimension.DIMENSION_TYPE_LINEAR)
-                ])
+                if config_command.exponential_inst_count:
+                    dimensions.add_dimensions(i, [
+                        SearchDimension(f"max_batch_size", SearchDimension.DIMENSION_TYPE_EXPONENTIAL),
+                        SearchDimension(f"instance_count", SearchDimension.DIMENSION_TYPE_EXPONENTIAL)
+                    ])
+                else:
+                    dimensions.add_dimensions(i, [
+                        SearchDimension(f"max_batch_size", SearchDimension.DIMENSION_TYPE_EXPONENTIAL),
+                        SearchDimension(f"instance_count", SearchDimension.DIMENSION_TYPE_LINEAR)
+                    ])
+
             #yapf: enable
 
             search_config = SearchConfig(
@@ -65,10 +73,11 @@ def create_generator(generator_name, config_command):
                 step_magnitude=config_command.magnitude,
                 min_initialized=config_command.min_initialized)
 
+            mvn = ModelVariantNameManager()
             generator = QuickRunConfigGenerator(search_config, config_command,
                                                 MagicMock(),
                                                 config_command.profile_models,
-                                                MagicMock())
+                                                MagicMock(), mvn)
             return generator
         else:
             raise Exception(f"Unknown generator {generator_name}")