Create detailed reports after profile (#691)

* Added new config option to create default detailed reports

* Adding documentation

* Detailed reports run after profile. Added unit testing

* Fix typo

* Changing to skip-detailed-reports

* Fixing errors in report.md documentation

* Changing detailed report sentence

Author: nv-braf

docs/config.md

@@ -230,6 +230,9 @@ cpu_only_composing_models: <comma-delimited-string-list>
 # Skips the generation of summary reports and tables
 [ skip_summary_reports: <bool> | default: false]
 
+# Skips the generation of detailed reports and tables
+[ skip_detailed_reports: <bool> | default: false]
+
 # Number of top configs to show in summary plots
 [ num_configs_per_model: <int> | default: 3]
 

docs/report.md

@@ -33,8 +33,8 @@ $ model-analyzer profile --profile-models <list of model names> --checkpoint-dir
 The export directory will, by default, contain 3 subdirectories. The summary
 report for a model will be located in `[export-path]/reports/summaries/<model name>`. The report will look like the one shown [_here_](../examples/online_summary.pdf).
 
-To disable summary report generation use `--summarize=false` or set the
-`summarize` yaml option to `false`.
+To disable summary report generation use `--skip-summary-reports` or set the
+`skip_summary_reports` yaml option to `false`.
 
 ## Detailed Reports
 
@@ -57,3 +57,7 @@ look like the one shown [_here_](../examples/online_detailed_report.pdf).
 See the [**configuring model
 analyzer**](./config.md) section for more details on how to configure these
 reports.
+
+Detailed reports for the top-N (based on the value of `--num-configs-per-model`) configurations are also generated at the end of `profile`.
+To disable detailed report generation after `profile` use `--skip-detailed-reports` or set the
+`skip_detailed_reports` yaml option to `false`.

model_analyzer/analyzer.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from typing import List, Union, Optional
-import sys
+import os, sys
 from model_analyzer.constants import LOGGER_NAME, PA_ERROR_LOG_FILENAME
 from .model_manager import ModelManager
 from .result.result_manager import ResultManager
@@ -126,17 +126,9 @@ def profile(self, client: TritonClient, gpus: List[GPUDevice], mode: str,
         if not self._config.skip_summary_reports:
             self._create_summary_tables(verbose)
             self._create_summary_reports(mode)
+            self._create_detailed_reports()
 
-            # TODO-TMA-650: Detailed reporting not supported for multi-model
-            if not self._config.run_config_profile_models_concurrently_enable:
-                for model in self._config.profile_models:
-                    logger.info(
-                        self._get_report_command_help_string(
-                            model.model_name()))
-
-            if self._metrics_manager.encountered_perf_analyzer_error():
-                logger.warning(f"Perf Analyzer encountered an error when profiling one or more configurations. " \
-                      f"See {self._config.export_path}/{PA_ERROR_LOG_FILENAME} for further details.\n")
+        self._check_for_perf_analyzer_errors()
 
     def report(self, mode: str) -> None:
         """
@@ -280,18 +272,28 @@ def _get_num_profiled_configs(self):
         ])
 
     def _get_report_command_help_string(self, model_name: str) -> str:
-        top_3_model_config_names = self._get_top_n_model_config_names(
-            n=3, model_name=model_name)
+        top_n_model_config_names = self._get_top_n_model_config_names(
+            n=self._config.num_configs_per_model, model_name=model_name)
         return (
             f'To generate detailed reports for the '
-            f'{len(top_3_model_config_names)} best {model_name} configurations, run '
-            f'`{self._get_report_command_string(top_3_model_config_names)}`')
+            f'{len(top_n_model_config_names)} best {model_name} configurations, run '
+            f'`{self._get_report_command_string(top_n_model_config_names)}`')
+
+    def _run_report_command(self, model_name: str) -> None:
+        top_n_model_config_names = self._get_top_n_model_config_names(
+            n=self._config.num_configs_per_model, model_name=model_name)
+        top_n_string = ','.join(top_n_model_config_names)
+        logger.info(
+            f'Generating detailed reports for the best configurations {top_n_string}:'
+        )
+        os.system(
+            f'{self._get_report_command_string(top_n_model_config_names)}')
 
     def _get_report_command_string(self,
-                                   top_3_model_config_names: List[str]) -> str:
+                                   top_n_model_config_names: List[str]) -> str:
         report_command_string = (f'model-analyzer report '
                                  f'--report-model-configs '
-                                 f'{",".join(top_3_model_config_names)}')
+                                 f'{",".join(top_n_model_config_names)}')
 
         if self._config.export_path is not None:
             report_command_string += (f' --export-path '
@@ -336,3 +338,19 @@ def _multiple_models_in_report_model_config(self) -> bool:
         ]
 
         return len(set(model_names)) > 1
+
+    def _check_for_perf_analyzer_errors(self) -> None:
+        if self._metrics_manager.encountered_perf_analyzer_error():
+            logger.warning(f"Perf Analyzer encountered an error when profiling one or more configurations. " \
+                    f"See {self._config.export_path}/{PA_ERROR_LOG_FILENAME} for further details.\n")
+
+    def _create_detailed_reports(self) -> None:
+        # TODO-TMA-650: Detailed reporting not supported for multi-model
+        if not self._config.run_config_profile_models_concurrently_enable:
+            for model in self._config.profile_models:
+                if not self._config.skip_detailed_reports:
+                    self._run_report_command(model.model_name())
+                else:
+                    logger.info(
+                        self._get_report_command_help_string(
+                            model.model_name()))

model_analyzer/config/input/config_command_profile.py

@@ -46,7 +46,8 @@
     DEFAULT_FILENAME_SERVER_ONLY, DEFAULT_NUM_CONFIGS_PER_MODEL, DEFAULT_NUM_TOP_MODEL_CONFIGS, \
     DEFAULT_INFERENCE_OUTPUT_FIELDS, DEFAULT_REQUEST_RATE_INFERENCE_OUTPUT_FIELDS, \
     DEFAULT_GPU_OUTPUT_FIELDS, DEFAULT_REQUEST_RATE_GPU_OUTPUT_FIELDS, DEFAULT_SERVER_OUTPUT_FIELDS, \
-    DEFAULT_ONLINE_OBJECTIVES, DEFAULT_ONLINE_PLOTS, DEFAULT_OFFLINE_PLOTS, DEFAULT_MODEL_WEIGHTING
+    DEFAULT_ONLINE_OBJECTIVES, DEFAULT_ONLINE_PLOTS, DEFAULT_OFFLINE_PLOTS, DEFAULT_MODEL_WEIGHTING, \
+    DEFAULT_SKIP_DETAILED_REPORTS
 
 from model_analyzer.constants import LOGGER_NAME
 from model_analyzer.triton.server.server_config import \
@@ -223,6 +224,15 @@ def _fill_config(self):
                 default_value=DEFAULT_SKIP_SUMMARY_REPORTS,
                 description=
                 'Skips the generation of analysis summary reports and tables.'))
+        self._add_config(
+            ConfigField(
+                'skip_detailed_reports',
+                flags=['--skip-detailed-reports'],
+                field_type=ConfigPrimitive(bool),
+                parser_args={'action': 'store_true'},
+                default_value=DEFAULT_SKIP_DETAILED_REPORTS,
+                description=
+                "Skips the generation of detailed summary reports and tables."))
 
         self._add_repository_configs()
         self._add_client_configs()

model_analyzer/config/input/config_defaults.py

@@ -33,6 +33,7 @@
 DEFAULT_LOG_LEVEL = 'INFO'
 DEFAULT_GPUS = 'all'
 DEFAULT_SKIP_SUMMARY_REPORTS = False
+DEFAULT_SKIP_DETAILED_REPORTS = False
 DEFAULT_OUTPUT_MODEL_REPOSITORY = os.path.join(os.getcwd(),
                                                'output_model_repository')
 DEFAULT_OVERRIDE_OUTPUT_REPOSITORY_FLAG = False

qa/L0_profile/test.sh

@@ -54,7 +54,6 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost:
 MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository"
 MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --filename-server-only=$FILENAME_SERVER_ONLY"
 MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL"
-MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-summary-reports"
 MODEL_ANALYZER_SUBCOMMAND="profile"
 run_analyzer
 if [ $? -ne 0 ]; then

tests/test_analyzer.py

@@ -64,7 +64,8 @@ def mock_get_list_of_models(self):
                     _create_metrics_manager=MagicMock(),
                     _create_model_manager=MagicMock(),
                     _get_server_only_metrics=MagicMock(),
-                    _profile_models=MagicMock())
+                    _profile_models=MagicMock(),
+                    _check_for_perf_analyzer_errors=MagicMock())
     def test_profile_skip_summary_reports(self, **mocks):
         """
         Tests when the skip_summary_reports config option is turned on,

tests/test_cli.py

@@ -67,6 +67,7 @@ def get_test_options():
         OptionStruct("bool", "profile","--reload-model-disable"),
         OptionStruct("bool", "profile","--early-exit-enable"),
         OptionStruct("bool", "profile","--skip-summary-reports"),
+        OptionStruct("bool", "profile","--skip-detailed-reports"),
         #Int/Float options
         # Options format:
         #   (int/float, MA step, long_option, short_option, test_value, expected_default_value)