Revert "Create detailed reports after profile (#691)" (#706)

This reverts commit b30e061.

Author: tgerdesnv

docs/config.md

@@ -230,9 +230,6 @@ cpu_only_composing_models: <comma-delimited-string-list>
 # Skips the generation of summary reports and tables
 [ skip_summary_reports: <bool> | default: false]
 
-# Skips the generation of detailed reports and tables
-[ skip_detailed_reports: <bool> | default: false]
-
 # Number of top configs to show in summary plots
 [ num_configs_per_model: <int> | default: 3]
 

docs/report.md

@@ -33,8 +33,8 @@ $ model-analyzer profile --profile-models <list of model names> --checkpoint-dir
 The export directory will, by default, contain 3 subdirectories. The summary
 report for a model will be located in `[export-path]/reports/summaries/<model name>`. The report will look like the one shown [_here_](../examples/online_summary.pdf).
 
-To disable summary report generation use `--skip-summary-reports` or set the
-`skip_summary_reports` yaml option to `false`.
+To disable summary report generation use `--summarize=false` or set the
+`summarize` yaml option to `false`.
 
 ## Detailed Reports
 
@@ -57,7 +57,3 @@ look like the one shown [_here_](../examples/online_detailed_report.pdf).
 See the [**configuring model
 analyzer**](./config.md) section for more details on how to configure these
 reports.
-
-Detailed reports for the top-N (based on the value of `--num-configs-per-model`) configurations are also generated at the end of `profile`.
-To disable detailed report generation after `profile` use `--skip-detailed-reports` or set the
-`skip_detailed_reports` yaml option to `false`.

model_analyzer/analyzer.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from typing import List, Union, Optional
-import os, sys
+import sys
 from model_analyzer.constants import LOGGER_NAME, PA_ERROR_LOG_FILENAME
 from .model_manager import ModelManager
 from .result.result_manager import ResultManager
@@ -126,9 +126,17 @@ def profile(self, client: TritonClient, gpus: List[GPUDevice], mode: str,
         if not self._config.skip_summary_reports:
             self._create_summary_tables(verbose)
             self._create_summary_reports(mode)
-            self._create_detailed_reports()
 
-        self._check_for_perf_analyzer_errors()
+            # TODO-TMA-650: Detailed reporting not supported for multi-model
+            if not self._config.run_config_profile_models_concurrently_enable:
+                for model in self._config.profile_models:
+                    logger.info(
+                        self._get_report_command_help_string(
+                            model.model_name()))
+
+            if self._metrics_manager.encountered_perf_analyzer_error():
+                logger.warning(f"Perf Analyzer encountered an error when profiling one or more configurations. " \
+                      f"See {self._config.export_path}/{PA_ERROR_LOG_FILENAME} for further details.\n")
 
     def report(self, mode: str) -> None:
         """
@@ -272,28 +280,18 @@ def _get_num_profiled_configs(self):
         ])
 
     def _get_report_command_help_string(self, model_name: str) -> str:
-        top_n_model_config_names = self._get_top_n_model_config_names(
-            n=self._config.num_configs_per_model, model_name=model_name)
+        top_3_model_config_names = self._get_top_n_model_config_names(
+            n=3, model_name=model_name)
         return (
             f'To generate detailed reports for the '
-            f'{len(top_n_model_config_names)} best {model_name} configurations, run '
-            f'`{self._get_report_command_string(top_n_model_config_names)}`')
-
-    def _run_report_command(self, model_name: str) -> None:
-        top_n_model_config_names = self._get_top_n_model_config_names(
-            n=self._config.num_configs_per_model, model_name=model_name)
-        top_n_string = ','.join(top_n_model_config_names)
-        logger.info(
-            f'Generating detailed reports for the best configurations {top_n_string}:'
-        )
-        os.system(
-            f'{self._get_report_command_string(top_n_model_config_names)}')
+            f'{len(top_3_model_config_names)} best {model_name} configurations, run '
+            f'`{self._get_report_command_string(top_3_model_config_names)}`')
 
     def _get_report_command_string(self,
-                                   top_n_model_config_names: List[str]) -> str:
+                                   top_3_model_config_names: List[str]) -> str:
         report_command_string = (f'model-analyzer report '
                                  f'--report-model-configs '
-                                 f'{",".join(top_n_model_config_names)}')
+                                 f'{",".join(top_3_model_config_names)}')
 
         if self._config.export_path is not None:
             report_command_string += (f' --export-path '
@@ -338,19 +336,3 @@ def _multiple_models_in_report_model_config(self) -> bool:
         ]
 
         return len(set(model_names)) > 1
-
-    def _check_for_perf_analyzer_errors(self) -> None:
-        if self._metrics_manager.encountered_perf_analyzer_error():
-            logger.warning(f"Perf Analyzer encountered an error when profiling one or more configurations. " \
-                    f"See {self._config.export_path}/{PA_ERROR_LOG_FILENAME} for further details.\n")
-
-    def _create_detailed_reports(self) -> None:
-        # TODO-TMA-650: Detailed reporting not supported for multi-model
-        if not self._config.run_config_profile_models_concurrently_enable:
-            for model in self._config.profile_models:
-                if not self._config.skip_detailed_reports:
-                    self._run_report_command(model.model_name())
-                else:
-                    logger.info(
-                        self._get_report_command_help_string(
-                            model.model_name()))

model_analyzer/config/input/config_command_profile.py

@@ -46,8 +46,7 @@
     DEFAULT_FILENAME_SERVER_ONLY, DEFAULT_NUM_CONFIGS_PER_MODEL, DEFAULT_NUM_TOP_MODEL_CONFIGS, \
     DEFAULT_INFERENCE_OUTPUT_FIELDS, DEFAULT_REQUEST_RATE_INFERENCE_OUTPUT_FIELDS, \
     DEFAULT_GPU_OUTPUT_FIELDS, DEFAULT_REQUEST_RATE_GPU_OUTPUT_FIELDS, DEFAULT_SERVER_OUTPUT_FIELDS, \
-    DEFAULT_ONLINE_OBJECTIVES, DEFAULT_ONLINE_PLOTS, DEFAULT_OFFLINE_PLOTS, DEFAULT_MODEL_WEIGHTING, \
-    DEFAULT_SKIP_DETAILED_REPORTS
+    DEFAULT_ONLINE_OBJECTIVES, DEFAULT_ONLINE_PLOTS, DEFAULT_OFFLINE_PLOTS, DEFAULT_MODEL_WEIGHTING
 
 from model_analyzer.constants import LOGGER_NAME
 from model_analyzer.triton.server.server_config import \
@@ -224,15 +223,6 @@ def _fill_config(self):
                 default_value=DEFAULT_SKIP_SUMMARY_REPORTS,
                 description=
                 'Skips the generation of analysis summary reports and tables.'))
-        self._add_config(
-            ConfigField(
-                'skip_detailed_reports',
-                flags=['--skip-detailed-reports'],
-                field_type=ConfigPrimitive(bool),
-                parser_args={'action': 'store_true'},
-                default_value=DEFAULT_SKIP_DETAILED_REPORTS,
-                description=
-                "Skips the generation of detailed summary reports and tables."))
 
         self._add_repository_configs()
         self._add_client_configs()

model_analyzer/config/input/config_defaults.py

@@ -33,7 +33,6 @@
 DEFAULT_LOG_LEVEL = 'INFO'
 DEFAULT_GPUS = 'all'
 DEFAULT_SKIP_SUMMARY_REPORTS = False
-DEFAULT_SKIP_DETAILED_REPORTS = False
 DEFAULT_OUTPUT_MODEL_REPOSITORY = os.path.join(os.getcwd(),
                                                'output_model_repository')
 DEFAULT_OVERRIDE_OUTPUT_REPOSITORY_FLAG = False

qa/L0_profile/test.sh

@@ -54,6 +54,7 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost:
 MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository"
 MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --filename-server-only=$FILENAME_SERVER_ONLY"
 MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL"
+MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-summary-reports"
 MODEL_ANALYZER_SUBCOMMAND="profile"
 run_analyzer
 if [ $? -ne 0 ]; then

tests/test_analyzer.py

@@ -64,8 +64,7 @@ def mock_get_list_of_models(self):
                     _create_metrics_manager=MagicMock(),
                     _create_model_manager=MagicMock(),
                     _get_server_only_metrics=MagicMock(),
-                    _profile_models=MagicMock(),
-                    _check_for_perf_analyzer_errors=MagicMock())
+                    _profile_models=MagicMock())
     def test_profile_skip_summary_reports(self, **mocks):
         """
         Tests when the skip_summary_reports config option is turned on,

tests/test_cli.py

@@ -67,7 +67,6 @@ def get_test_options():
         OptionStruct("bool", "profile","--reload-model-disable"),
         OptionStruct("bool", "profile","--early-exit-enable"),
         OptionStruct("bool", "profile","--skip-summary-reports"),
-        OptionStruct("bool", "profile","--skip-detailed-reports"),
         #Int/Float options
         # Options format:
         #   (int/float, MA step, long_option, short_option, test_value, expected_default_value)