brute working

Author: tgerdesnv

model_analyzer/config/generate/brute_plus_binary_parameter_search_run_config_generator.py

@@ -132,9 +132,11 @@ def _binary_search_over_top_results(self) -> Generator[RunConfig, None, None]:
             for result in top_results:
                 run_config = deepcopy(result.run_config())
                 model_parameters = self._get_model_parameters(model_name)
+                perf_analyzer_flags = self._get_model_perf_analyzer_flags(model_name)
                 parameter_search = ParameterSearch(
                     config=self._config,
                     model_parameters=model_parameters,
+                    perf_analyzer_flags=perf_analyzer_flags,
                     skip_parameter_sweep=True,
                 )
                 for parameter in parameter_search.search_parameters():
@@ -151,6 +153,11 @@ def _get_model_parameters(self, model_name: str) -> Dict:
 
         return {}
 
+    def _get_model_perf_analyzer_flags(self, model_name: str) -> Dict:
+        for model in self._models:
+            if model_name == model.model_name():
+                return model.perf_analyzer_flags()
+
     def _set_parameter(
         self, run_config: RunConfig, model_parameters: Dict, parameter: int
     ) -> RunConfig:

model_analyzer/config/generate/perf_analyzer_config_generator.py

@@ -172,7 +172,9 @@ def _create_parameter_list(self) -> List[int]:
         # The two possible parameters are request rate or concurrency
         # Concurrency is the default and will be used unless the user specifies
         # request rate, either as a model parameter or a config option
-        if self._cli_config.is_request_rate_specified(self._model_parameters):
+        if "request-intervals" in self._perf_analyzer_flags:
+            return [self._perf_analyzer_flags["request-intervals"]]
+        elif self._cli_config.is_request_rate_specified(self._model_parameters):
             return self._create_request_rate_list()
         else:
             return self._create_concurrency_list()
@@ -207,6 +209,7 @@ def _generate_perf_configs(self) -> None:
         for params in utils.generate_parameter_combinations(
             perf_config_non_parameter_values
         ):
+            # FIXME variable name
             configs_with_concurrency = []
             for parameter in self._parameters:
                 new_perf_config = PerfAnalyzerConfig()
@@ -217,7 +220,9 @@ def _generate_perf_configs(self) -> None:
 
                 new_perf_config.update_config(params)
 
-                if self._cli_config.is_request_rate_specified(self._model_parameters):
+                if "request-intervals" in self._perf_analyzer_flags:
+                    pass
+                elif self._cli_config.is_request_rate_specified(self._model_parameters):
                     new_perf_config.update_config({"request-rate-range": parameter})
                 else:
                     new_perf_config.update_config({"concurrency-range": parameter})

model_analyzer/perf_analyzer/perf_config.py

@@ -273,6 +273,7 @@ def extract_model_specific_parameters(self):
             "batch-size": self._options["-b"],
             "concurrency-range": self._args["concurrency-range"],
             "request-rate-range": self._args["request-rate-range"],
+            "request-intervals": self._args["request-intervals"],
         }
 
     @classmethod

model_analyzer/plots/detailed_plot.py

@@ -89,7 +89,6 @@ def __init__(self, name, title, bar_width=0.5):
         self._fig.set_figheight(8)
         self._fig.set_figwidth(12)
 
-        self._ax_latency.set_xlabel("Concurrent Client Requests")
         self._ax_latency.set_ylabel(latency_axis_label)
         self._ax_throughput.set_ylabel(throughput_axis_label)
 
@@ -144,6 +143,18 @@ def add_run_config_measurement(self, run_config_measurement):
                 ]
             )
 
+        if (
+            "request-intervals" in run_config_measurement.model_specific_pa_params()[0]
+            and run_config_measurement.model_specific_pa_params()[0][
+                "request-intervals"
+            ]
+        ):
+            self._data["request-intervals"].append(
+                run_config_measurement.model_specific_pa_params()[0][
+                    "request-intervals"
+                ]
+            )
+
         self._data["perf_throughput"].append(
             run_config_measurement.get_non_gpu_metric_value(tag="perf_throughput")
         )
@@ -164,19 +175,20 @@ def plot_data(self):
         on this plot's Axes object
         """
 
-        # Need to change the default x-axis plot title for request rates
-        if "request_rate" in self._data and self._data["request_rate"][0]:
+        # Update the x-axis plot title
+        if "request-intervals" in self._data and self._data["request-intervals"][0]:
+            self._ax_latency.set_xlabel("Request Intervals File")
+            sort_indices_key = "request-intervals"
+        elif "request_rate" in self._data and self._data["request_rate"][0]:
             self._ax_latency.set_xlabel("Client Request Rate")
-
-        # Sort the data by request rate or concurrency
-        if "request_rate" in self._data and self._data["request_rate"][0]:
-            sort_indices = list(
-                zip(*sorted(enumerate(self._data["request_rate"]), key=lambda x: x[1]))
-            )[0]
+            sort_indices_key = "request_rate"
         else:
-            sort_indices = list(
-                zip(*sorted(enumerate(self._data["concurrency"]), key=lambda x: x[1]))
-            )[0]
+            self._ax_latency.set_xlabel("Concurrent Client Requests")
+            sort_indices_key = "concurrency"
+
+        sort_indices = list(
+            zip(*sorted(enumerate(self._data[sort_indices_key]), key=lambda x: x[1]))
+        )[0]
 
         sorted_data = {
             key: [data_list[i] for i in sort_indices]
@@ -197,10 +209,7 @@ def plot_data(self):
         )
         bottoms = None
 
-        if "request_rate" in self._data:
-            sorted_data["indices"] = list(map(str, sorted_data["request_rate"]))
-        else:
-            sorted_data["indices"] = list(map(str, sorted_data["concurrency"]))
+        sorted_data["indices"] = list(map(str, sorted_data[sort_indices_key]))
 
         # Plot latency breakdown with concurrency casted as string to make uniform x
         for metric, label in labels.items():

model_analyzer/record/metrics_manager.py

@@ -753,7 +753,11 @@ def _get_triton_metrics_gpus(self):
     def _print_run_config_info(self, run_config):
         for model_run_config in run_config.model_run_configs():
             perf_config = model_run_config.perf_config()
-            if perf_config["request-rate-range"]:
+            if perf_config["request-intervals"]:
+                logger.info(
+                    f"Profiling {model_run_config.model_variant_name()}: client batch size={perf_config['batch-size']}, request-intervals={perf_config['request-intervals']}"
+                )
+            elif perf_config["request-rate-range"]:
                 logger.info(
                     f"Profiling {model_run_config.model_variant_name()}: client batch size={perf_config['batch-size']}, request-rate-range={perf_config['request-rate-range']}"
                 )

model_analyzer/result/parameter_search.py

@@ -45,6 +45,7 @@ def __init__(
         self,
         config: ConfigCommandProfile,
         model_parameters: dict = {},
+        perf_analyzer_flags: dict = {},
         skip_parameter_sweep: bool = False,
     ) -> None:
         """
@@ -60,6 +61,8 @@ def __init__(
             model_parameters
         )
 
+        self._inference_load_is_custom = "request-intervals" in perf_analyzer_flags
+
         if self._parameter_is_request_rate:
             self._min_parameter_index = int(
                 log2(config.run_config_search_min_request_rate)
@@ -98,10 +101,11 @@ def search_parameters(self) -> Generator[int, None, None]:
         a binary parameter search around the point where the constraint
         violated
         """
-        yield from self._perform_parameter_sweep()
+        if not self._inference_load_is_custom:
+            yield from self._perform_parameter_sweep()
 
-        if self._was_constraint_violated():
-            yield from self._perform_binary_parameter_search()
+            if self._was_constraint_violated():
+                yield from self._perform_binary_parameter_search()
 
     def _perform_parameter_sweep(self) -> Generator[int, None, None]:
         for parameter in (