Detailed reporting after profile (#711)

* Cherry picked changes

* Changing to evaluate a mock config and directly call report

* Switch away from using mock

* Adding type checking to CLI.parse()

* Restoring original profile config

* Adding missing newline

* Fixing codeQL issue

Author: nv-braf

docs/config.md

@@ -230,6 +230,9 @@ cpu_only_composing_models: <comma-delimited-string-list>
 # Skips the generation of summary reports and tables
 [ skip_summary_reports: <bool> | default: false]
 
+# Skips the generation of detailed reports and tables
+[ skip_detailed_reports: <bool> | default: false]
+
 # Number of top configs to show in summary plots
 [ num_configs_per_model: <int> | default: 3]
 

docs/report.md

@@ -33,8 +33,8 @@ $ model-analyzer profile --profile-models <list of model names> --checkpoint-dir
 The export directory will, by default, contain 3 subdirectories. The summary
 report for a model will be located in `[export-path]/reports/summaries/<model name>`. The report will look like the one shown [_here_](../examples/online_summary.pdf).
 
-To disable summary report generation use `--summarize=false` or set the
-`summarize` yaml option to `false`.
+To disable summary report generation use `--skip-summary-reports` or set the
+`skip_summary_reports` yaml option to `false`.
 
 ## Detailed Reports
 
@@ -57,3 +57,7 @@ look like the one shown [_here_](../examples/online_detailed_report.pdf).
 See the [**configuring model
 analyzer**](./config.md) section for more details on how to configure these
 reports.
+
+Detailed reports for the top-N (based on the value of `--num-configs-per-model`) configurations are also generated at the end of `profile`.
+To disable detailed report generation after `profile` use `--skip-detailed-reports` or set the
+`skip_detailed_reports` yaml option to `false`.

model_analyzer/analyzer.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from typing import List, Union, Optional
+from copy import deepcopy
 import sys
 from model_analyzer.constants import LOGGER_NAME, PA_ERROR_LOG_FILENAME
 from .model_manager import ModelManager
@@ -32,6 +33,8 @@
 from model_analyzer.state.analyzer_state_manager import AnalyzerStateManager
 from model_analyzer.triton.server.server import TritonServer
 
+from model_analyzer.cli.cli import CLI
+
 from model_analyzer.config.generate.base_model_config_generator import BaseModelConfigGenerator
 
 from .triton.client.client import TritonClient
@@ -126,17 +129,9 @@ def profile(self, client: TritonClient, gpus: List[GPUDevice], mode: str,
         if not self._config.skip_summary_reports:
             self._create_summary_tables(verbose)
             self._create_summary_reports(mode)
+            self._create_detailed_reports(mode)
 
-            # TODO-TMA-650: Detailed reporting not supported for multi-model
-            if not self._config.run_config_profile_models_concurrently_enable:
-                for model in self._config.profile_models:
-                    logger.info(
-                        self._get_report_command_help_string(
-                            model.model_name()))
-
-            if self._metrics_manager.encountered_perf_analyzer_error():
-                logger.warning(f"Perf Analyzer encountered an error when profiling one or more configurations. " \
-                      f"See {self._config.export_path}/{PA_ERROR_LOG_FILENAME} for further details.\n")
+        self._check_for_perf_analyzer_errors()
 
     def report(self, mode: str) -> None:
         """
@@ -280,18 +275,35 @@ def _get_num_profiled_configs(self):
         ])
 
     def _get_report_command_help_string(self, model_name: str) -> str:
-        top_3_model_config_names = self._get_top_n_model_config_names(
-            n=3, model_name=model_name)
+        top_n_model_config_names = self._get_top_n_model_config_names(
+            n=self._config.num_configs_per_model, model_name=model_name)
         return (
             f'To generate detailed reports for the '
-            f'{len(top_3_model_config_names)} best {model_name} configurations, run '
-            f'`{self._get_report_command_string(top_3_model_config_names)}`')
+            f'{len(top_n_model_config_names)} best {model_name} configurations, run '
+            f'`{self._get_report_command_string(top_n_model_config_names)}`')
+
+    def _run_report_command(self, model_name: str, mode: str) -> None:
+        top_n_model_config_names = self._get_top_n_model_config_names(
+            n=self._config.num_configs_per_model, model_name=model_name)
+        top_n_string = ','.join(top_n_model_config_names)
+        logger.info(
+            f'Generating detailed reports for the best configurations {top_n_string}:'
+        )
+
+        # [1:] removes 'model-analyzer' from the args
+        args = self._get_report_command_string(top_n_model_config_names).split(
+            ' ')[1:]
+
+        original_profile_config = deepcopy(self._config)
+        self._config = self._create_report_config(args)
+        self.report(mode)
+        self._config = original_profile_config
 
     def _get_report_command_string(self,
-                                   top_3_model_config_names: List[str]) -> str:
+                                   top_n_model_config_names: List[str]) -> str:
         report_command_string = (f'model-analyzer report '
                                  f'--report-model-configs '
-                                 f'{",".join(top_3_model_config_names)}')
+                                 f'{",".join(top_n_model_config_names)}')
 
         if self._config.export_path is not None:
             report_command_string += (f' --export-path '
@@ -336,3 +348,26 @@ def _multiple_models_in_report_model_config(self) -> bool:
         ]
 
         return len(set(model_names)) > 1
+
+    def _check_for_perf_analyzer_errors(self) -> None:
+        if self._metrics_manager.encountered_perf_analyzer_error():
+            logger.warning(f"Perf Analyzer encountered an error when profiling one or more configurations. " \
+                    f"See {self._config.export_path}/{PA_ERROR_LOG_FILENAME} for further details.\n")
+
+    def _create_detailed_reports(self, mode: str) -> None:
+        # TODO-TMA-650: Detailed reporting not supported for multi-model
+        if not self._config.run_config_profile_models_concurrently_enable:
+            for model in self._config.profile_models:
+                if not self._config.skip_detailed_reports:
+                    self._run_report_command(model.model_name(), mode)
+                else:
+                    logger.info(
+                        self._get_report_command_help_string(
+                            model.model_name()))
+
+    def _create_report_config(self, args: list) -> ConfigCommandReport:
+        config = ConfigCommandReport()
+        cli = CLI()
+        cli.add_subcommand(cmd='report', help="", config=config)
+        cli.parse(args)
+        return config

model_analyzer/cli/cli.py

@@ -12,9 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List, Union, Optional, Tuple
+
 import logging
 import argparse
-from argparse import ArgumentParser
+from argparse import ArgumentParser, Namespace
+
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.input.config_command_report import ConfigCommandReport
+
 from model_analyzer.constants import LOGGER_NAME
 
 logger = logging.getLogger(LOGGER_NAME)
@@ -118,11 +124,20 @@ def _add_config_arguments(self, subparser, config):
                     **config_field.parser_args(),
                 )
 
-    def parse(self):
+    def parse(
+        self,
+        input_args: Optional[List] = None
+    ) -> Tuple[Namespace, Union[ConfigCommandProfile, ConfigCommandReport]]:
         """
         Parse CLI options using ArgumentParsers 
         and set config values.
 
+        Parameters
+        ----------
+        input_args: List
+            The list of arguments to be parsed 
+            (if None then command line arguments will be used)
+
         Returns
         -------
         args : Namespace
@@ -133,7 +148,8 @@ def parse(self):
             already filled in with values from CLI or YAML.
         """
 
-        args = self._parser.parse_args()
+        args = self._parser.parse_args(input_args)
+
         if args.subcommand is None:
             self._parser.print_help()
             self._parser.exit()

model_analyzer/config/input/config_command_profile.py

@@ -46,7 +46,8 @@
     DEFAULT_FILENAME_SERVER_ONLY, DEFAULT_NUM_CONFIGS_PER_MODEL, DEFAULT_NUM_TOP_MODEL_CONFIGS, \
     DEFAULT_INFERENCE_OUTPUT_FIELDS, DEFAULT_REQUEST_RATE_INFERENCE_OUTPUT_FIELDS, \
     DEFAULT_GPU_OUTPUT_FIELDS, DEFAULT_REQUEST_RATE_GPU_OUTPUT_FIELDS, DEFAULT_SERVER_OUTPUT_FIELDS, \
-    DEFAULT_ONLINE_OBJECTIVES, DEFAULT_ONLINE_PLOTS, DEFAULT_OFFLINE_PLOTS, DEFAULT_MODEL_WEIGHTING
+    DEFAULT_ONLINE_OBJECTIVES, DEFAULT_ONLINE_PLOTS, DEFAULT_OFFLINE_PLOTS, DEFAULT_MODEL_WEIGHTING, \
+    DEFAULT_SKIP_DETAILED_REPORTS
 
 from model_analyzer.constants import LOGGER_NAME
 from model_analyzer.triton.server.server_config import \
@@ -223,6 +224,15 @@ def _fill_config(self):
                 default_value=DEFAULT_SKIP_SUMMARY_REPORTS,
                 description=
                 'Skips the generation of analysis summary reports and tables.'))
+        self._add_config(
+            ConfigField(
+                'skip_detailed_reports',
+                flags=['--skip-detailed-reports'],
+                field_type=ConfigPrimitive(bool),
+                parser_args={'action': 'store_true'},
+                default_value=DEFAULT_SKIP_DETAILED_REPORTS,
+                description=
+                "Skips the generation of detailed summary reports and tables."))
 
         self._add_repository_configs()
         self._add_client_configs()

model_analyzer/config/input/config_defaults.py

@@ -33,6 +33,7 @@
 DEFAULT_LOG_LEVEL = 'INFO'
 DEFAULT_GPUS = 'all'
 DEFAULT_SKIP_SUMMARY_REPORTS = False
+DEFAULT_SKIP_DETAILED_REPORTS = False
 DEFAULT_OUTPUT_MODEL_REPOSITORY = os.path.join(os.getcwd(),
                                                'output_model_repository')
 DEFAULT_OVERRIDE_OUTPUT_REPOSITORY_FLAG = False

qa/L0_profile/test.sh

@@ -53,6 +53,7 @@ MODEL_ANALYZER_BASE_ARGS="$MODEL_ANALYZER_BASE_ARGS --output-model-repository-pa
 MODEL_ANALYZER_BASE_ARGS="$MODEL_ANALYZER_BASE_ARGS --filename-server-only=$FILENAME_SERVER_ONLY"
 MODEL_ANALYZER_BASE_ARGS="$MODEL_ANALYZER_BASE_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL"
 MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_BASE_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY"
+MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_BASE_ARGS --skip-summary-reports"
 MODEL_ANALYZER_SUBCOMMAND="profile"
 
 run_analyzer

tests/test_analyzer.py

@@ -64,7 +64,8 @@ def mock_get_list_of_models(self):
                     _create_metrics_manager=MagicMock(),
                     _create_model_manager=MagicMock(),
                     _get_server_only_metrics=MagicMock(),
-                    _profile_models=MagicMock())
+                    _profile_models=MagicMock(),
+                    _check_for_perf_analyzer_errors=MagicMock())
     def test_profile_skip_summary_reports(self, **mocks):
         """
         Tests when the skip_summary_reports config option is turned on,

tests/test_cli.py

@@ -67,6 +67,7 @@ def get_test_options():
         OptionStruct("bool", "profile","--reload-model-disable"),
         OptionStruct("bool", "profile","--early-exit-enable"),
         OptionStruct("bool", "profile","--skip-summary-reports"),
+        OptionStruct("bool", "profile","--skip-detailed-reports"),
         #Int/Float options
         # Options format:
         #   (int/float, MA step, long_option, short_option, test_value, expected_default_value)