Adding concurrency sweep after quick search completes (#508)

* Parity with quick run

* Concurrency sweep working. Need to refactor

* Refactored get_configs()

* Refactored

* Added flag to brute RCG to skip default config generation

* Fixed unit tests and added default skip to quick + concurrency sweep

* Removing type

* Adding newline

* Adding config return to methods

* Adding custom __setattr__ to config command

* Setting attr correctly in _set_search_mode

* Setting value correctly in config command

* Using user specified num of top configs

* Adding TODO with story

* Fixes based on review comments

* Fixing type checking issues

Author: nv-braf

model_analyzer/config/generate/brute_run_config_generator.py

@@ -12,20 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List, Optional
+
 from .config_generator_interface import ConfigGeneratorInterface
 from model_analyzer.config.run.run_config import RunConfig
 from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
 from model_analyzer.config.generate.model_run_config_generator import ModelRunConfigGenerator
 from model_analyzer.config.generate.model_variant_name_manager import ModelVariantNameManager
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
 
 
 class BruteRunConfigGenerator(ConfigGeneratorInterface):
     """
     Generates all RunConfigs to execute via brute force given a list of models
     """
 
-    def __init__(self, config, gpus, models, client,
-                 model_variant_name_manager):
+    def __init__(self,
+                 config,
+                 gpus,
+                 models,
+                 client,
+                 model_variant_name_manager,
+                 skip_default_config: bool = False):
         """
         Parameters
         ----------
@@ -39,6 +47,8 @@ def __init__(self, config, gpus, models, client,
         client: TritonClient
         
         model_variant_name_manager: ModelVariantNameManager
+        
+        skip_default_config: bool
         """
         self._config = config
         self._gpus = gpus
@@ -52,9 +62,11 @@ def __init__(self, config, gpus, models, client,
         self._num_models = len(models)
 
         self._curr_model_run_configs = [None for n in range(self._num_models)]
-        self._curr_results = [[] for n in range(self._num_models)]
+        self._curr_results: List = [[] for n in range(self._num_models)]
         self._curr_generators = [None for n in range(self._num_models)]
 
+        self._skip_default_config = skip_default_config
+
     def set_last_results(self, measurements):
         for index in range(self._num_models):
             self._curr_results[index].extend(measurements)
@@ -70,7 +82,9 @@ def get_configs(self):
         yield from self._get_next_config()
 
     def _get_next_config(self):
-        yield from self._generate_subset(0, default_only=True)
+        if not self._skip_default_config:
+            yield from self._generate_subset(0, default_only=True)
+
         if self._should_generate_non_default_configs():
             yield from self._generate_subset(0, default_only=False)
 

model_analyzer/config/generate/model_variant_name_manager.py

@@ -55,12 +55,12 @@ def get_model_variant_name(self, model_name: str, model_config_dict: Dict,
 
         variant_found, model_variant_name = self._find_existing_variant(mcd)
 
-        if variant_found:
-            return (True, model_variant_name)
-
         if self._is_default_config(param_combo):
             return (False, model_name + '_config_default')
 
+        if variant_found:
+            return (True, model_variant_name)
+
         model_variant_name = self._create_new_model_variant(model_name, mcd)
 
         return (False, model_variant_name)

model_analyzer/config/generate/quick_plus_concurrency_sweep_run_config_generator.py

@@ -0,0 +1,198 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Union, Optional, Generator
+
+from .config_generator_interface import ConfigGeneratorInterface
+
+from model_analyzer.config.generate.base_model_config_generator import BaseModelConfigGenerator
+from model_analyzer.config.generate.search_config import SearchConfig
+from model_analyzer.config.generate.coordinate import Coordinate
+from model_analyzer.config.generate.coordinate_data import CoordinateData
+from model_analyzer.config.generate.neighborhood import Neighborhood
+from model_analyzer.config.generate.brute_run_config_generator import BruteRunConfigGenerator
+from model_analyzer.config.generate.quick_run_config_generator import QuickRunConfigGenerator
+from model_analyzer.config.generate.model_variant_name_manager import ModelVariantNameManager
+from model_analyzer.config.run.model_run_config import ModelRunConfig
+from model_analyzer.config.run.run_config import RunConfig
+from model_analyzer.perf_analyzer.perf_config import PerfAnalyzerConfig
+from model_analyzer.triton.model.model_config import ModelConfig
+from model_analyzer.triton.client.client import TritonClient
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.input.objects.config_model_profile_spec import ConfigModelProfileSpec
+from model_analyzer.result.result_manager import ResultManager
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+from model_analyzer.record.metrics_manager import MetricsManager
+from model_analyzer.result.results import Results
+from model_analyzer.result.run_config_result import RunConfigResult
+
+from model_analyzer.constants import LOGGER_NAME, MAGNITUDE_DECAY_RATE
+from model_analyzer.config.input.config_defaults import DEFAULT_NUM_CONFIGS_PER_MODEL, \
+    DEFAULT_RUN_CONFIG_MIN_CONCURRENCY, DEFAULT_RUN_CONFIG_MAX_CONCURRENCY
+
+from copy import deepcopy
+
+import logging
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class QuickPlusConcurrencySweepRunConfigGenerator(ConfigGeneratorInterface):
+    """
+    First run QuickRunConfigGenerator for a hill climbing search, then use 
+    Brute for a concurrency sweep of the default and Top N results
+    """
+
+    def __init__(self, search_config: SearchConfig,
+                 config: ConfigCommandProfile, gpus: List[GPUDevice],
+                 models: List[ConfigModelProfileSpec], client: TritonClient,
+                 result_manager: ResultManager,
+                 model_variant_name_manager: ModelVariantNameManager):
+        """
+        Parameters
+        ----------
+        search_config: SearchConfig
+            Defines parameters and dimensions for the search
+        config: ConfigCommandProfile
+            Profile configuration information
+        gpus: List of GPUDevices
+        models: List of ConfigModelProfileSpec
+            List of models to profile
+        client: TritonClient
+        result_manager: ResultManager
+            The object that handles storing and sorting the results from the perf analyzer
+        model_variant_name_manager: ModelVariantNameManager
+            Maps model variants to config names
+        
+        model_variant_name_manager: ModelVariantNameManager
+        """
+        self._search_config = search_config
+        self._config = config
+        self._gpus = gpus
+        self._models = models
+        self._client = client
+        self._result_manager = result_manager
+        self._model_variant_name_manager = model_variant_name_manager
+
+    def set_last_results(self,
+                         measurements: List[Optional[RunConfigMeasurement]]):
+        self._rcg.set_last_results(measurements)
+
+    def get_configs(self) -> Generator[RunConfig, None, None]:
+        """
+        Returns
+        -------
+        RunConfig
+            The next RunConfig generated by this class
+        """
+        yield from self._execute_quick_search()
+        yield from self._sweep_concurrency_over_top_results()
+
+    def _execute_quick_search(self):
+        self._rcg = self._create_quick_run_config_generator()
+
+        yield from self._rcg.get_configs()
+
+    def _create_quick_run_config_generator(self) -> QuickRunConfigGenerator:
+        return QuickRunConfigGenerator(
+            search_config=self._search_config,
+            config=self._config,
+            gpus=self._gpus,
+            models=self._models,
+            client=self._client,
+            model_variant_name_manager=self._model_variant_name_manager)
+
+    def _sweep_concurrency_over_top_results(self):
+        top_results = self._result_manager.top_n_results(
+            n=self._config.num_top_model_configs)
+
+        for count, result in enumerate(top_results):
+            new_config = self._create_new_config_command_profile(result)
+            self._rcg = self._create_brute_run_config_generator(
+                new_config, skip_default_config=(count != 0))
+
+            yield from self._rcg.get_configs()
+
+    def _create_new_config_command_profile(
+            self, result: RunConfigResult) -> ConfigCommandProfile:
+        new_config = deepcopy(self._config)
+
+        new_config = self._set_search_mode(new_config)
+        new_config = self._set_parameters(result, new_config)
+
+        return new_config
+
+    def _create_brute_run_config_generator(
+            self, new_config: ConfigCommandProfile,
+            skip_default_config: bool) -> BruteRunConfigGenerator:
+        return BruteRunConfigGenerator(
+            config=new_config,
+            gpus=self._gpus,
+            models=self._models,
+            client=self._client,
+            model_variant_name_manager=self._model_variant_name_manager,
+            skip_default_config=skip_default_config)
+
+    def _set_search_mode(self,
+                         config: ConfigCommandProfile) -> ConfigCommandProfile:
+        config.run_config_search_mode = 'brute'
+        config.run_config_search_disable = False
+        config.early_exit_enable = True
+
+        return config
+
+    def _set_parameters(self, result: RunConfigResult,
+                        config: ConfigCommandProfile) -> ConfigCommandProfile:
+        batch_size = self._find_batch_size(result)
+        config = self._set_batch_size(config, batch_size)
+
+        instance_count = self._find_instance_count(result)
+        config = self._set_instance_count(config, instance_count)
+
+        config = self._set_concurrency(config)
+
+        return config
+
+    # TODO: TMA-879 - This approach isn't going to work for multi-model
+    # We will need to create a yaml config to set each models
+    # batch size/instances seperately
+    def _find_batch_size(self, result: RunConfigResult) -> int:
+        return result.run_config().model_run_configs()[0].model_config(
+        ).get_config()['max_batch_size']
+
+    def _find_instance_count(self, result: RunConfigResult) -> int:
+        return result.run_config().model_run_configs()[0].model_config(
+        ).get_config()['instance_group'][0]['count']
+
+    def _set_batch_size(self, config: ConfigCommandProfile,
+                        batch_size: int) -> ConfigCommandProfile:
+        config.run_config_search_min_model_batch_size = batch_size
+        config.run_config_search_max_model_batch_size = batch_size
+
+        return config
+
+    def _set_instance_count(self, config: ConfigCommandProfile,
+                            instance_count: int) -> ConfigCommandProfile:
+        config.run_config_search_min_instance_count = instance_count
+        config.run_config_search_max_instance_count = instance_count
+
+        return config
+
+    def _set_concurrency(self,
+                         config: ConfigCommandProfile) -> ConfigCommandProfile:
+        config.run_config_search_min_concurrency = DEFAULT_RUN_CONFIG_MIN_CONCURRENCY
+        config.run_config_search_max_concurrency = DEFAULT_RUN_CONFIG_MAX_CONCURRENCY
+
+        return config

model_analyzer/config/generate/quick_run_config_generator.py

@@ -46,12 +46,9 @@ class QuickRunConfigGenerator(ConfigGeneratorInterface):
     Hill climbing algorithm to create RunConfigs
     """
 
-    def __init__(self,
-                 search_config: SearchConfig,
-                 config: ConfigCommandProfile,
-                 gpus: List[GPUDevice],
-                 models: List[ConfigModelProfileSpec],
-                 client: TritonClient,
+    def __init__(self, search_config: SearchConfig,
+                 config: ConfigCommandProfile, gpus: List[GPUDevice],
+                 models: List[ConfigModelProfileSpec], client: TritonClient,
                  model_variant_name_manager: ModelVariantNameManager):
         """
         Parameters
@@ -124,7 +121,8 @@ def _step(self):
         Determine self._coordinate_to_measure, which is what is used to
         create the next RunConfig
         """
-        if self._measuring_home_coordinate() and self._get_last_results() is None:
+        if self._measuring_home_coordinate(
+        ) and self._get_last_results() is None:
             self._take_step_back()
         elif self._neighborhood.enough_coordinates_initialized():
             self._take_step()
@@ -206,8 +204,7 @@ def _take_step_back(self):
             coordinate_in=self._best_coordinate)
 
         logger.debug(
-            f"Stepping back: {self._home_coordinate}->{new_coordinate}"
-        )
+            f"Stepping back: {self._home_coordinate}->{new_coordinate}")
         self._home_coordinate = new_coordinate
         self._coordinate_to_measure = new_coordinate
         self._recreate_neighborhood()
@@ -242,8 +239,7 @@ def _get_starting_coordinate(self) -> Coordinate:
         min_indexes = self._search_config.get_min_indexes()
         return Coordinate(min_indexes)
 
-    def _get_coordinate_values(self,
-                               coordinate: Coordinate,
+    def _get_coordinate_values(self, coordinate: Coordinate,
                                key: int) -> Dict[str, Union[int, float]]:
         dims = self._search_config.get_dimensions()
         values = dims.get_values_for_coordinate(coordinate)
@@ -282,9 +278,6 @@ def _get_next_model_config(self, model_num: int) -> ModelConfig:
             'instance_group': [{
                 'count': dimension_values['instance_count'],
                 'kind': "KIND_GPU",
-                'rate_limiter': {
-                    'priority': 1
-                }
             }]
         }
 
@@ -298,8 +291,7 @@ def _get_next_model_config(self, model_num: int) -> ModelConfig:
             model_variant_name_manager=self._model_variant_name_manager)
         return model_config
 
-    def _get_next_perf_analyzer_config(self,
-                                       model_variant_name: str,
+    def _get_next_perf_analyzer_config(self, model_variant_name: str,
                                        model_num: int) -> PerfAnalyzerConfig:
         dimension_values = self._get_coordinate_values(
             self._coordinate_to_measure, model_num)
@@ -338,8 +330,8 @@ def _print_debug_logs(self, measurements: List[Union[RunConfigMeasurement,
                 best_latency = self._best_measurement.get_non_gpu_metric_value(
                     "perf_latency_p99")
             else:
-                best_throughput = None
-                best_latency = None
+                best_throughput = 0
+                best_latency = 0
 
             logger.debug(
                 f"Measurement for {self._coordinate_to_measure}: "
@@ -348,5 +340,4 @@ def _print_debug_logs(self, measurements: List[Union[RunConfigMeasurement,
             )
         else:
             logger.debug(
-                f"Measurement for {self._coordinate_to_measure}: None."
-            )
+                f"Measurement for {self._coordinate_to_measure}: None.")