More experiment sweep changes (#509)

tgerdesnv · web-flow · commit cdb5f77046bf · 2022-09-01T10:45:43.000-05:00
* Changes to sweeping experiments of quick search

* minor PR feedback

* Fix constructor after rebase
diff --git a/experiments/checkpoint_experiment_data.py b/experiments/checkpoint_experiment_data.py
@@ -16,13 +16,16 @@
 from model_analyzer.state.analyzer_state_manager import AnalyzerStateManager
 from unittest.mock import MagicMock
 from copy import deepcopy
+import re
 
 
 class CheckpointExperimentData(ExperimentData):
     """
     Extends ExperimentData to be able to preload data from a checkpoint
     """
 
+    LOAD_ONLY_VISABLE = True
+
     def __init__(self, config):
         super().__init__()
         self._default_run_config = None
@@ -66,6 +69,10 @@ def _load_checkpoint(self, config):
                     metric_objectives=[config.objectives])
                 pa_key = self._make_pa_key_from_cli_string(perf_analyzer_string)
 
+                if CheckpointExperimentData.LOAD_ONLY_VISABLE:
+                    if not self._are_keys_visable_to_algorithm(ma_key, pa_key):
+                        continue
+
                 existing_measurement = self._get_run_config_measurement_from_keys(
                     ma_key, pa_key, skip_warn=True)
                 if not existing_measurement or run_config_measurement > existing_measurement:
@@ -79,10 +86,18 @@ def _load_checkpoint(self, config):
         self._print_map()
 
     def _print_map(self):
-        for i in range(0, 10):
-            row_str = ""
+        row_str = "\t\t"
+        for j in range(0, 10):
+            row_str += f"    [{j}]\t\t"
+        print(row_str)
+
+        has_exponential_inst_count = self._has_exponential_inst_count()
+
+        for i in range(0, 100):
+            row_has_data = False
+            row_str = f"\t[{i}]"
             for j in range(0, 10):
-                instance_count = j + 1
+                instance_count = 2**j if has_exponential_inst_count else j + 1
                 max_batch_size = 2**i
 
                 ma_key = f"instance_count={instance_count},max_batch_size={max_batch_size}"
@@ -97,9 +112,38 @@ def _print_map(self):
                 tput = 0
                 lat = 0
                 if measurement:
+                    row_has_data = True
                     tput = measurement.get_non_gpu_metric_value(
                         'perf_throughput')
                     lat = measurement.get_non_gpu_metric_value(
                         'perf_latency_p99')
                 row_str += f"\t{tput:4.1f}:{lat:4.1f}"
-            print(row_str)
+
+            # Print at least the first 7 rows, and then stop when we hit an empty row
+            # (Some databases don't have data for the first 6 rows)
+            if row_has_data or i < 7:
+                print(row_str)
+            else:
+                break
+
+    def _has_exponential_inst_count(self) -> bool:
+        # See if instance count of 3 is in the database. If not, it is exponential (1,2,4,8)
+        ret = False
+        ma_key = f"instance_count=3,max_batch_size=1"
+        pa_key = "8"
+        measurement = self._get_run_config_measurement_from_keys(ma_key,
+                                                                 pa_key,
+                                                                 skip_warn=True)
+        if not measurement:
+            ret = True
+        return ret
+
+    def _are_keys_visable_to_algorithm(self, ma_key, pa_key) -> bool:
+        # The quick algorithm can only see meaurements where the
+        # concurrency is 2 * inst_count * max_batch_size.
+        results = re.search("instance_count=(\d+),max_batch_size=(\d+)", ma_key)
+        inst_count = int(results.group(1))
+        max_batch_size = int(results.group(2))
+        expected_pa_key = self._clamp_to_power_of_two(2 * inst_count *
+                                                      max_batch_size)
+        return expected_pa_key == int(pa_key)
diff --git a/experiments/config_command_experiment.py b/experiments/config_command_experiment.py
@@ -46,6 +46,13 @@ def _fill_config(self):
                         flags=['--magnitude'],
                         default_value=20,
                         description='The size of each step'))
+        self._add_config(
+            ConfigField('min_mbs_index',
+                        field_type=ConfigPrimitive(int),
+                        flags=['--min-mbs-index'],
+                        default_value=0,
+                        description='The minimum index for max batch size'))
+
         self._add_config(
             ConfigField(
                 'min_initialized',
diff --git a/experiments/evaluate_config_generator.py b/experiments/evaluate_config_generator.py
@@ -19,6 +19,8 @@
 from checkpoint_experiment_data import CheckpointExperimentData
 from experiment_file_writer import ExperimentFileWriter
 from unittest.mock import MagicMock, patch
+from model_analyzer.state.analyzer_state import AnalyzerState
+from model_analyzer.config.generate.model_variant_name_manager import ModelVariantNameManager
 
 
 class EvaluateConfigGenerator:
@@ -28,6 +30,9 @@ class EvaluateConfigGenerator:
     """
 
     def __init__(self, model_name, data_path, output_path, other_args):
+
+        self._patch_checkpoint_load()
+
         self._output_path = output_path
         self._model_name = model_name
         self._config_command = ExperimentConfigCommandCreator.make_config(
@@ -43,16 +48,17 @@ def __init__(self, model_name, data_path, output_path, other_args):
             MagicMock(return_value=self._default_config_dict))
         p.start()
 
-    def execute_generator(self, generator_name):
+    def execute_generator(self):
 
         generator = GeneratorExperimentFactory.create_generator(
-            generator_name, self._config_command)
+            self._config_command)
 
         self._run_generator(generator)
 
     def print_results(self):
         result_evaluator = ExperimentEvaluator(self._checkpoint_data,
-                                               self._profile_data)
+                                               self._profile_data,
+                                               self._config_command)
         result_evaluator.print_results()
 
     def store_results(self):
@@ -78,3 +84,18 @@ def _run_generator(self, cg):
                 run_config, run_config_measurement)
 
             cg.set_last_results([run_config_measurement])
+
+    def _patch_checkpoint_load(self):
+
+        old_fn = AnalyzerState.from_dict
+
+        def patched_analyzer_state_from_dict(state_dict):
+            if 'ModelManager.model_variant_name_manager' not in state_dict:
+                state_dict[
+                    "ModelManager.model_variant_name_manager"] = ModelVariantNameManager(
+                    )
+            return old_fn(state_dict)
+
+        p = patch('model_analyzer.state.analyzer_state.AnalyzerState.from_dict',
+                  patched_analyzer_state_from_dict)
+        p.start()
diff --git a/experiments/experiment_evaluator.py b/experiments/experiment_evaluator.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from experiment_data import ExperimentData
+from experiments.config_command_experiment import ConfigCommandExperiment
 
 
 class ExperimentEvaluator:
@@ -21,10 +22,15 @@ class ExperimentEvaluator:
     a checkpoint of raw data
     """
 
-    def __init__(self, raw_data: ExperimentData, profile_data: ExperimentData):
+    def __init__(self, raw_data: ExperimentData, profile_data: ExperimentData,
+                 config_command: ConfigCommandExperiment):
         self._raw_data = raw_data
         self._profile_data = profile_data
 
+        self._maximize_throughput = True
+        if config_command.min_throughput is not None:
+            self._maximize_throughput = False
+
     def print_results(self):
         overall_best_measurement = self._raw_data.get_best_run_config_measurement(
         )
@@ -69,7 +75,13 @@ def print_results(self):
                 'perf_latency_p99')
             overall_best_throughput = overall_best_measurement.get_non_gpu_metric_value(
                 'perf_throughput')
-            percentile = round(best_throughput / overall_best_throughput, 2)
+            overall_best_latency = overall_best_measurement.get_non_gpu_metric_value(
+                'perf_latency_p99')
+
+            if self._maximize_throughput:
+                percentile = round(best_throughput / overall_best_throughput, 2)
+            else:
+                percentile = round(overall_best_latency / best_latency, 2)
         else:
             best_throughput = None
             best_latency = None
diff --git a/experiments/generator_experiment_factory.py b/experiments/generator_experiment_factory.py
@@ -19,12 +19,15 @@
 from model_analyzer.config.generate.search_dimension import SearchDimension
 from model_analyzer.config.generate.search_dimensions import SearchDimensions
 from unittest.mock import MagicMock, patch
+from model_analyzer.config.generate.run_config_generator_factory import RunConfigGeneratorFactory
 
 
 class GeneratorExperimentFactory:
 
+    command_config = None
+
     @staticmethod
-    def create_generator(generator_name, config_command):
+    def create_generator(config_command):
         """ 
         Create and return a RunConfig generator of the requested name
         
@@ -39,45 +42,53 @@ def create_generator(generator_name, config_command):
             The config for model analyzer algorithm experiment
         """
 
-        if generator_name == "BruteRunConfigGenerator":
-            generator = BruteRunConfigGenerator(config_command, MagicMock(),
-                                                config_command.profile_models,
-                                                MagicMock())
-            p = patch(
-                'model_analyzer.config.generate.brute_run_config_generator.BruteRunConfigGenerator.determine_triton_server_env'
-            )
-            p.start()
-
-            return generator
-        elif generator_name == "QuickRunConfigGenerator":
-            dimensions = SearchDimensions()
+        GeneratorExperimentFactory.config_command = config_command
 
-            #yapf: disable
-            for i, _ in enumerate(config_command.profile_models):
-                if config_command.exponential_inst_count:
-                    dimensions.add_dimensions(i, [
-                        SearchDimension(f"max_batch_size", SearchDimension.DIMENSION_TYPE_EXPONENTIAL),
-                        SearchDimension(f"instance_count", SearchDimension.DIMENSION_TYPE_EXPONENTIAL)
-                    ])
-                else:
-                    dimensions.add_dimensions(i, [
-                        SearchDimension(f"max_batch_size", SearchDimension.DIMENSION_TYPE_EXPONENTIAL),
-                        SearchDimension(f"instance_count", SearchDimension.DIMENSION_TYPE_LINEAR)
-                    ])
+        p1 = patch(
+            'model_analyzer.config.generate.run_config_generator_factory.RunConfigGeneratorFactory._get_batching_supported_dimensions',
+            GeneratorExperimentFactory.get_batching_supported_dimensions)
+        p2 = patch(
+            'model_analyzer.config.generate.run_config_generator_factory.RunConfigGeneratorFactory._get_batching_not_supported_dimensions',
+            GeneratorExperimentFactory.get_batching_not_supported_dimensions)
+        p1.start()
+        p2.start()
+        mvn = ModelVariantNameManager()
+        generator = RunConfigGeneratorFactory.create_run_config_generator(
+            config_command, MagicMock(), config_command.profile_models,
+            MagicMock(), MagicMock(), mvn)
+        return generator
 
-            #yapf: enable
+    @staticmethod
+    def get_batching_supported_dimensions():
+        mbs_min = GeneratorExperimentFactory.config_command.min_mbs_index
+        ret = [
+            SearchDimension(f"max_batch_size",
+                            SearchDimension.DIMENSION_TYPE_EXPONENTIAL, mbs_min)
+        ]
+        if GeneratorExperimentFactory.config_command.exponential_inst_count:
+            ret.append(
+                SearchDimension(f"instance_count",
+                                SearchDimension.DIMENSION_TYPE_EXPONENTIAL))
+        else:
+            ret.append(
+                SearchDimension(f"instance_count",
+                                SearchDimension.DIMENSION_TYPE_LINEAR))
+        return ret
 
-            search_config = SearchConfig(
-                dimensions=dimensions,
-                radius=config_command.radius,
-                step_magnitude=config_command.magnitude,
-                min_initialized=config_command.min_initialized)
+    @staticmethod
+    def get_batching_not_supported_dimensions():
+        mbs_min = GeneratorExperimentFactory.config_command.min_mbs_index
 
-            mvn = ModelVariantNameManager()
-            generator = QuickRunConfigGenerator(search_config, config_command,
-                                                MagicMock(),
-                                                config_command.profile_models,
-                                                MagicMock(), mvn)
-            return generator
+        ret = [
+            SearchDimension(f"concurrency",
+                            SearchDimension.DIMENSION_TYPE_EXPONENTIAL, mbs_min)
+        ]
+        if GeneratorExperimentFactory.config_command.exponential_inst_count:
+            ret.append(
+                SearchDimension(f"instance_count",
+                                SearchDimension.DIMENSION_TYPE_EXPONENTIAL))
         else:
-            raise Exception(f"Unknown generator {generator_name}")
+            ret.append(
+                SearchDimension(f"instance_count",
+                                SearchDimension.DIMENSION_TYPE_LINEAR))
+        return ret
diff --git a/experiments/main.py b/experiments/main.py
@@ -49,10 +49,6 @@
                     required=False,
                     default="./output",
                     help="The path to the output csv files")
-parser.add_argument("--generator",
-                    type=str,
-                    required=True,
-                    help="The name of the config generator to evaluate")
 args, other_args = parser.parse_known_args()
 
 if args.verbose:
@@ -63,7 +59,7 @@
 
 ecg = EvaluateConfigGenerator(args.model_name, args.data_path, args.output_path,
                               other_args)
-ecg.execute_generator(args.generator)
+ecg.execute_generator()
 ecg.print_results()
 
 if args.save:
diff --git a/experiments/scripts/minimize_latency.yml b/experiments/scripts/minimize_latency.yml
@@ -0,0 +1,16 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+objectives:
+  - perf_latency_p99
diff --git a/experiments/scripts/sweep_checkpoints.py b/experiments/scripts/sweep_checkpoints.py
diff --git a/model_analyzer/config/generate/run_config_generator_factory.py b/model_analyzer/config/generate/run_config_generator_factory.py
