Offline mode for analyze and report (#164)

* Offline mode for analyze and report

* Review edits

* Fixed online mode defaults

Author: Ashwin Ramesh

docs/cli.md

@@ -24,7 +24,18 @@ $ model-analyzer -h
 ```
 
 Options like `-q`, `--quiet` and `-v`, `--verbose` are global and apply to all
-model analyzer subcommands. 
+model analyzer subcommands.
+
+## Model Analyze Modes
+
+The `-m` or `--mode` flag is global and is accessible to all subcommands. It tells the model analyzer the context
+in which it is being run. Currently model analyzer supports 2 modes.
+
+### Online Mode
+
+This is the default mode. When in this mode, Model Analyzer will operate to find the optimal model
+configuration for an online inference scenario. In this scenario, Triton server will receive requests
+on demand with an expectation that latency will be minimized.
 
 ## Model Analyzer Subcommands
 

model_analyzer/analyzer.py

@@ -118,12 +118,17 @@ def profile(self, client):
             f"Finished profiling. Obtained measurements for models: {profiled_model_list}."
         )
 
-    def analyze(self):
+    def analyze(self, mode):
         """
         subcommand: ANALYZE
 
         Constructs results from measurements,
         sorts them, and dumps them to tables.
+
+        Parameters
+        ----------
+        mode : str
+            Global mode that the analyzer is running on
         """
 
         if not isinstance(self._config, ConfigCommandAnalyze):
@@ -136,6 +141,7 @@ def analyze(self):
         if not gpu_info:
             gpu_info = {}
         self._report_manager = ReportManager(
+            mode=mode,
             config=self._config,
             gpu_info=gpu_info,
             result_manager=self._result_manager)
@@ -155,12 +161,17 @@ def analyze(self):
         self._result_manager.tabulate_results()
         self._result_manager.write_and_export_results()
 
-    def report(self):
+    def report(self, mode):
         """
         Subcommand: REPORT
 
         Generates detailed information on
         one or more model configs
+
+        Parameters
+        ----------
+        mode : str
+            Global mode that the analyzer is running on
         """
 
         if not isinstance(self._config, ConfigCommandReport):
@@ -173,6 +184,7 @@ def report(self):
         if not gpu_info:
             gpu_info = {}
         self._report_manager = ReportManager(
+            mode=mode,
             config=self._config,
             result_manager=self._result_manager,
             gpu_info=gpu_info)

model_analyzer/cli/cli.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
 import logging
 import argparse
 from argparse import ArgumentParser
@@ -24,7 +23,6 @@ class CLI:
     """
     CLI class to parse the commandline arguments
     """
-
     def __init__(self):
         self._parser = ArgumentParser()
         self._add_global_options()
@@ -36,9 +34,8 @@ def __init__(self):
 
     def _add_global_options(self):
         """
-        Retrieves the arguments from the command line and loads them into an
-        ArgumentParser. It will also configure the Model Analyzer config
-        accordingly.
+        Adds the Model Analyzer's global options
+        to the parser
         """
 
         self._parser.add_argument(
@@ -54,6 +51,7 @@ def _add_global_options(self):
         self._parser.add_argument('-m',
                                   '--mode',
                                   type=str,
+                                  default='online',
                                   choices=['online', 'offline'],
                                   help='Choose a preset configuration mode.')
 
@@ -100,8 +98,8 @@ def _add_config_arguments(self, subparser, config):
             # 'store_true' and 'store_false' does not
             # allow 'type' or 'choices' parameters
             if 'action' in parser_args and (
-                    parser_args['action'] == 'store_true' or
-                    parser_args['action'] == 'store_false'):
+                    parser_args['action'] == 'store_true'
+                    or parser_args['action'] == 'store_false'):
                 subparser.add_argument(
                     *config.flags(),
                     default=argparse.SUPPRESS,

model_analyzer/config/input/config_command_analyze.py

@@ -13,12 +13,12 @@
 # limitations under the License.
 
 from .config_defaults import \
-    DEFAULT_ANALYSIS_PLOTS, DEFAULT_CHECKPOINT_DIRECTORY, DEFAULT_EXPORT_PATH, \
+    DEFAULT_CHECKPOINT_DIRECTORY, DEFAULT_EXPORT_PATH, \
     DEFAULT_FILENAME_MODEL_GPU, DEFAULT_FILENAME_MODEL_INFERENCE, \
     DEFAULT_FILENAME_SERVER_ONLY, DEFAULT_GPU_OUTPUT_FIELDS, \
     DEFAULT_INFERENCE_OUTPUT_FIELDS, DEFAULT_NUM_CONFIGS_PER_MODEL, \
-    DEFAULT_NUM_TOP_MODEL_CONFIGS, DEFAULT_OBJECTIVES, \
-    DEFAULT_SERVER_OUTPUT_FIELDS, DEFAULT_SUMMARIZE_FLAG
+    DEFAULT_NUM_TOP_MODEL_CONFIGS, DEFAULT_OFFLINE_OBJECTIVES, DEFAULT_ONLINE_ANALYSIS_PLOTS, \
+    DEFAULT_OFFLINE_ANALYSIS_PLOTS, DEFAULT_ONLINE_OBJECTIVES,DEFAULT_SERVER_OUTPUT_FIELDS, DEFAULT_SUMMARIZE_FLAG
 from .config_field import ConfigField
 from .config_object import ConfigObject
 from .config_union import ConfigUnion
@@ -97,7 +97,7 @@ def objective_list_output_mapper(objectives):
             ConfigField(
                 'objectives',
                 field_type=objectives_scheme,
-                default_value=DEFAULT_OBJECTIVES,
+                default_value=DEFAULT_OFFLINE_OBJECTIVES,
                 description=
                 'Model Analyzer uses the objectives described here to find the best configuration for each model.'
             ))
@@ -276,6 +276,13 @@ def _add_shorthand_configs(self):
                 description=
                 "Shorthand flag for specifying a maximum latency in ms."))
 
+        self._add_config(
+            ConfigField('min_throughput',
+                        flags=['--min-throughput'],
+                        field_type=ConfigPrimitive(int),
+                        description=
+                        "Shorthand flag for specifying a minimum throughput."))
+
     def _preprocess_and_verify_arguments(self):
         """
         Enforces some rules on the config.
@@ -318,11 +325,18 @@ def set_config_values(self, args):
             this exception
         """
 
+        if args.mode == 'online' and 'latency_budget' not in args:
+            self._fields['objectives'].set_default_value(
+                DEFAULT_ONLINE_OBJECTIVES)
+
         super().set_config_values(args)
 
         # Add plot configs and after config parse. User should not be able to edit these plots
         self._add_plot_configs()
-        self._fields['plots'].set_value(DEFAULT_ANALYSIS_PLOTS)
+        if args.mode == 'online':
+            self._fields['plots'].set_value(DEFAULT_ONLINE_ANALYSIS_PLOTS)
+        elif args.mode == 'offline':
+            self._fields['plots'].set_value(DEFAULT_OFFLINE_ANALYSIS_PLOTS)
 
     def _add_plot_configs(self):
         """
@@ -369,6 +383,18 @@ def _autofill_values(self):
                         'max': self.latency_budget
                     }})
 
+        # Set global constraints if latency budget is specified
+        if self.min_throughput:
+            if self.constraints:
+                constraints = self.constraints
+                constraints['perf_throughput'] = {'min': self.min_throughput}
+                self._fields['constraints'].set_value(constraints)
+            else:
+                self._fields['constraints'].set_value(
+                    {'perf_throughput': {
+                        'min': self.min_throughput
+                    }})
+
         new_analysis_models = {}
         for model in self.analysis_models:
             new_model = {}
@@ -397,5 +423,17 @@ def _autofill_values(self):
                             'max': self.latency_budget
                         }
                     }
+
+            if self.min_throughput:
+                if 'constraints' in new_model:
+                    new_model['constraints']['perf_throughput'] = {
+                        'min': self.min_throughput
+                    }
+                else:
+                    new_model['constraints'] = {
+                        'perf_throughput': {
+                            'min': self.min_throughput
+                        }
+                    }
             new_analysis_models[model.model_name()] = new_model
         self._fields['analysis_models'].set_value(new_analysis_models)

model_analyzer/config/input/config_command_profile.py

@@ -29,8 +29,8 @@
 from .config_defaults import \
     DEFAULT_BATCH_SIZES, DEFAULT_CHECKPOINT_DIRECTORY, \
     DEFAULT_CLIENT_PROTOCOL, DEFAULT_DURATION_SECONDS, \
-    DEFAULT_GPUS, DEFAULT_LOG_LEVEL, DEFAULT_MAX_RETRIES, \
-    DEFAULT_MONITORING_INTERVAL, DEFAULT_OBJECTIVES, \
+    DEFAULT_GPUS, DEFAULT_MAX_RETRIES, \
+    DEFAULT_MONITORING_INTERVAL, DEFAULT_OFFLINE_OBJECTIVES, \
     DEFAULT_OUTPUT_MODEL_REPOSITORY, DEFAULT_OVERRIDE_OUTPUT_REPOSITORY_FLAG, \
     DEFAULT_PERF_ANALYZER_CPU_UTIL, DEFAULT_PERF_ANALYZER_PATH, DEFAULT_PERF_ANALYZER_TIMEOUT, \
     DEFAULT_PERF_OUTPUT_FLAG, DEFAULT_RUN_CONFIG_MAX_CONCURRENCY, \
@@ -287,7 +287,7 @@ def objective_list_output_mapper(objectives):
             ConfigField(
                 'objectives',
                 field_type=objectives_scheme,
-                default_value=DEFAULT_OBJECTIVES,
+                default_value=DEFAULT_OFFLINE_OBJECTIVES,
                 description=
                 'Model Analyzer uses the objectives described here to find the best configuration for each model.'
             ))

model_analyzer/config/input/config_command_report.py

@@ -20,7 +20,8 @@
 from .config_list_generic import ConfigListGeneric
 from .config_list_string import ConfigListString
 from .config_defaults import \
-    DEFAULT_CHECKPOINT_DIRECTORY, DEFAULT_EXPORT_PATH, DEFAULT_REPORT_FORMAT, DEFAULT_REPORT_PLOTS
+    DEFAULT_CHECKPOINT_DIRECTORY, DEFAULT_EXPORT_PATH, \
+    DEFAULT_OFFLINE_REPORT_PLOTS, DEFAULT_ONLINE_REPORT_PLOTS, DEFAULT_REPORT_FORMAT
 from .config_field import ConfigField
 from .config_primitive import ConfigPrimitive
 from .config_command import ConfigCommand
@@ -36,7 +37,6 @@ class ConfigCommandReport(ConfigCommand):
     """
     Model Analyzer config object.
     """
-
     def _fill_config(self):
         """
         Builder function makes calls to add config to 
@@ -59,21 +59,22 @@ def _fill_config(self):
                 "Full path to directory to which to read and write checkpoints and profile data."
             ))
         self._add_config(
-            ConfigField('export_path',
-                        flags=['--export-path', '-e'],
-                        default_value=DEFAULT_EXPORT_PATH,
-                        field_type=ConfigPrimitive(str),
-                        description=
-                        "Full path to directory in which to store the results"))
+            ConfigField(
+                'export_path',
+                flags=['--export-path', '-e'],
+                default_value=DEFAULT_EXPORT_PATH,
+                field_type=ConfigPrimitive(str),
+                description=
+                "Full path to directory in which to store the results"))
         plots_scheme = ConfigObject(schema={
             '*':
-                ConfigObject(
-                    schema={
-                        'title': ConfigPrimitive(type_=str),
-                        'x_axis': ConfigPrimitive(type_=str),
-                        'y_axis': ConfigPrimitive(type_=str),
-                        'monotonic': ConfigPrimitive(type_=bool)
-                    })
+            ConfigObject(
+                schema={
+                    'title': ConfigPrimitive(type_=str),
+                    'x_axis': ConfigPrimitive(type_=str),
+                    'y_axis': ConfigPrimitive(type_=str),
+                    'monotonic': ConfigPrimitive(type_=bool)
+                })
         },
                                     output_mapper=ConfigPlot.from_object)
         self._add_config(
@@ -84,7 +85,7 @@ def _fill_config(self):
                     ConfigListGeneric(type_=plots_scheme,
                                       output_mapper=ConfigPlot.from_list)
                 ]),
-                default_value=DEFAULT_REPORT_PLOTS,
+                default_value=DEFAULT_ONLINE_REPORT_PLOTS,
                 description=
                 'Model analyzer uses the information in this section to construct plots of the results.'
             ))
@@ -131,6 +132,33 @@ def _fill_config(self):
                         ]),
                         description='Output file format for detailed report.'))
 
+    def set_config_values(self, args):
+        """
+        Set the config values. This function sets all the values for the
+        config. CLI arguments have the highest priority, then YAML config
+        values and then default values.
+
+        Parameters
+        ----------
+        args : argparse.Namespace
+            Parsed arguments from the CLI
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+            If the required fields are not specified, it will raise
+            this exception
+        """
+
+        if args.mode == 'online':
+            self._fields['plots'].set_default_value(
+                DEFAULT_ONLINE_REPORT_PLOTS)
+        elif args.mode == 'offline':
+            self._fields['plots'].set_default_value(
+                DEFAULT_OFFLINE_REPORT_PLOTS)
+
+        super().set_config_values(args)
+
     def _preprocess_and_verify_arguments(self):
         """
         Enforces some rules on the config.
@@ -167,7 +195,8 @@ def _autofill_values(self):
                         'x_axis': plot.x_axis(),
                         'y_axis': plot.y_axis(),
                         'monotonic': plot.monotonic()
-                    } for plot in self.plots
+                    }
+                    for plot in self.plots
                 }
             else:
                 new_report_model_config['plots'] = {
@@ -176,10 +205,12 @@ def _autofill_values(self):
                         'x_axis': plot.x_axis(),
                         'y_axis': plot.y_axis(),
                         'monotonic': plot.monotonic()
-                    } for plot in model.plots()
+                    }
+                    for plot in model.plots()
                 }
 
             new_report_model_configs[
                 model.model_config_name()] = new_report_model_config
 
-        self._fields['report_model_configs'].set_value(new_report_model_configs)
+        self._fields['report_model_configs'].set_value(
+            new_report_model_configs)