Updated detailed report and results columns (#161)

* Updated detailed report

* Review edits, fixed report mode for CPU only

* Review edits - report info

Author: Ashwin Ramesh

docs/metrics.md

@@ -78,4 +78,4 @@ just metrics) that should be displayed in the output tables.
   dynamic batcher
 * `satisfies_constraints`: `Yes` if this measurement satisfies constraints, `No`
   otherwise.
-* `gpu_id`: The id of the GPU this measurement was taken on.
+* `gpu_uuid`: The UUID of the GPU this measurement was taken on.

model_analyzer/analyzer.py

@@ -35,7 +35,6 @@ class Analyzer:
     model_analyzer. Configured with metrics to monitor, exposes profiling and
     result writing methods.
     """
-
     def __init__(self, config, server, state_manager):
         """
         Parameters
@@ -84,7 +83,8 @@ def profile(self, client):
             config=self._config,
             client=client,
             server=self._server,
-            result_manager=self._result_manager)
+            result_manager=self._result_manager,
+            state_manager=self._state_manager)
 
         self._model_manager = ModelManager(
             config=self._config,
@@ -131,8 +131,14 @@ def analyze(self):
                 f"Expected config of type {ConfigCommandAnalyze}, got {type(self._config)}."
             )
 
+        gpu_info = self._state_manager.get_state_variable(
+            'MetricsManager.gpus')
+        if not gpu_info:
+            gpu_info = {}
         self._report_manager = ReportManager(
-            config=self._config, result_manager=self._result_manager)
+            config=self._config,
+            gpu_info=gpu_info,
+            result_manager=self._result_manager)
 
         # Create result tables, put top results and get stats
         dcgm_metrics, perf_metrics, cpu_metrics = \
@@ -162,8 +168,14 @@ def report(self):
                 f"Expected config of type {ConfigCommandReport}, got {type(self._config)}."
             )
 
+        gpu_info = self._state_manager.get_state_variable(
+            'MetricsManager.gpus')
+        if not gpu_info:
+            gpu_info = {}
         self._report_manager = ReportManager(
-            config=self._config, result_manager=self._result_manager)
+            config=self._config,
+            result_manager=self._result_manager,
+            gpu_info=gpu_info)
 
         self._report_manager.create_detailed_reports()
         self._report_manager.export_detailed_reports()

model_analyzer/config/input/config_defaults.py

@@ -86,12 +86,12 @@
     'perf_throughput', 'perf_latency', 'cpu_used_ram'
 ]
 DEFAULT_GPU_OUTPUT_FIELDS = [
-    'model_name', 'gpu_id', 'batch_size', 'concurrency', 'model_config_path',
+    'model_name', 'gpu_uuid', 'batch_size', 'concurrency', 'model_config_path',
     'instance_group', 'dynamic_batch_sizes', 'satisfies_constraints',
     'gpu_used_memory', 'gpu_utilization', 'gpu_power_usage'
 ]
 DEFAULT_SERVER_OUTPUT_FIELDS = [
-    'model_name', 'gpu_id', 'gpu_used_memory', 'gpu_utilization',
+    'model_name', 'gpu_uuid', 'gpu_used_memory', 'gpu_utilization',
     'gpu_power_usage'
 ]
 

model_analyzer/plots/plot_manager.py

@@ -29,7 +29,6 @@ class PlotManager:
     This class manages the construction and arrangement
     of plots generated by model analyzer
     """
-
     def __init__(self, config, result_manager):
         """
         Parameters
@@ -140,8 +139,8 @@ def create_detailed_plots(self):
             model_config_name = model.model_config_name()
             self._detailed_plots[model_config_name] = DetailedPlot(
                 f'latency_breakdown', 'Online Performance')
-            measurements = self._result_manager.get_model_config_measurements(
-                model_config_name)[1]
+            model_config, measurements = self._result_manager.get_model_config_measurements(
+                model_config_name)
 
             # If model_config_name was present in results
             if measurements:
@@ -152,6 +151,10 @@ def create_detailed_plots(self):
 
             # Create the simple plots for the detailed reports
             for plot_config in model.plots():
+                if model_config.cpu_only() and (
+                        plot_config.y_axis().startswith('gpu_')
+                        or plot_config.x_axis().startswith('gpu_')):
+                    continue
                 self._create_update_simple_plot(plots_key=model_config_name,
                                                 plot_config=plot_config,
                                                 measurements=measurements,
@@ -183,8 +186,8 @@ def export_detailed_plots(self):
             os.makedirs(detailed_model_config_plot_dir, exist_ok=True)
             plot.save(detailed_model_config_plot_dir)
 
-            simple_model_config_plot_dir = os.path.join(simple_plot_dir,
-                                                        model_config_name)
+            simple_model_config_plot_dir = os.path.join(
+                simple_plot_dir, model_config_name)
             os.makedirs(simple_model_config_plot_dir, exist_ok=True)
             for plot in self._simple_plots[model_config_name].values():
                 plot.save(simple_model_config_plot_dir)

model_analyzer/record/metrics_manager.py

@@ -44,7 +44,7 @@ class MetricsManager:
         "gpu_power_usage"
     ]
 
-    def __init__(self, config, client, server, result_manager):
+    def __init__(self, config, client, server, result_manager, state_manager):
         """
         Parameters
         ----------
@@ -58,19 +58,45 @@ def __init__(self, config, client, server, result_manager):
         result_manager : ResultManager
             instance that manages the result tables and 
             adding results
+        state_manager: AnalyzerStateManager
+            manages the analyzer state
         """
 
         self._config = config
         self._client = client
         self._server = server
         self._result_manager = result_manager
+        self._state_manager = state_manager
 
         self._dcgm_metrics, self._perf_metrics, self._cpu_metrics = \
              MetricsManager.categorize_metrics()
         self._gpus = GPUDeviceFactory.verify_requested_gpus(self._config.gpus)
+        self._init_state()
 
-        self._dcgm_monitor = None
-        self._cpu_monitor = None
+    def _init_state(self):
+        """
+        Sets MetricsManager object managed
+        state variables in AnalyerState
+        """
+
+        gpu_info = self._state_manager.get_state_variable(
+            'MetricsManager.gpu_info')
+
+        if self._state_manager.starting_fresh_run() or gpu_info is None:
+            gpu_info = {}
+
+        for i in range(len(self._gpus)):
+            if self._gpus[i] not in gpu_info:
+                device_info = {}
+                device = numba.cuda.list_devices()[i]
+                device_info['name'] = device.name
+                with device:
+                    # convert bytes to GB
+                    device_info['total_memory'] = numba.cuda.current_context(
+                    ).get_memory_info().total
+                gpu_info[self._gpus[i]] = device_info
+
+        self._state_manager.set_state_variable('MetricsManager.gpus', gpu_info)
 
     @classmethod
     def categorize_metrics(cls):
@@ -86,7 +112,7 @@ def categorize_metrics(cls):
 
         dcgm_metrics, perf_metrics, cpu_metrics = [], [], []
         # Separates metrics and objectives into related lists
-        for metric in MetricsManager.get_metric_types(cls.metric_tags):
+        for metric in MetricsManager.get_metric_types(tags=cls.metric_tags):
             if metric in DCGMMonitor.model_analyzer_to_dcgm_field:
                 dcgm_metrics.append(metric)
             elif metric in PerfAnalyzer.perf_metrics:
@@ -147,7 +173,7 @@ def profile_model(self, run_config, perf_output_writer=None):
         else:
             perf_analyzer_metrics = perf_analyzer_metrics_or_status
 
-        # Get metrics for model inference and combine metrics that do not have GPU ID
+        # Get metrics for model inference and combine metrics that do not have GPU UUID
         model_gpu_metrics = {}
         if not cpu_only:
             model_gpu_metrics = self._get_gpu_inference_metrics()
@@ -273,12 +299,13 @@ def _get_gpu_inference_metrics(self):
 
         records_groupby_gpu = {}
         records_groupby_gpu = dcgm_record_aggregator.groupby(
-            self._dcgm_metrics, lambda record: record.device().device_id())
+            self._dcgm_metrics, lambda record: str(
+                record.device().device_uuid(), encoding='ascii'))
 
         gpu_metrics = defaultdict(list)
         for _, metric in records_groupby_gpu.items():
-            for gpu_id, metric_value in metric.items():
-                gpu_metrics[gpu_id].append(metric_value)
+            for gpu_uuid, metric_value in metric.items():
+                gpu_metrics[gpu_uuid].append(metric_value)
 
         return gpu_metrics
 

model_analyzer/reports/pdf_report.py

@@ -23,7 +23,6 @@ class PDFReport(Report):
     constructed in html and
     written to disk as a PDF
     """
-
     def __init__(self):
         self._head = ""
         self._body = ""
@@ -72,7 +71,11 @@ def add_subheading(self, subheading):
 
         self._body += f'<h3>{subheading}</h3>'
 
-    def add_images(self, images, image_captions, image_width=100):
+    def add_images(self,
+                   images,
+                   image_captions,
+                   image_width=100,
+                   float="center"):
         """
         Parameters
         ----------
@@ -84,13 +87,15 @@ def add_images(self, images, image_captions, image_width=100):
         image_width: int 
             Percentage of the the row of images
             will occupy.
+        float: str
+            Alignment of the div containing each image in the row
         """
 
         image_row = ""
         for img, caption in zip(images, image_captions):
             with open(img, "rb") as image_file:
                 data_uri = base64.b64encode(image_file.read()).decode('ascii')
-                image_row += f"<div class=\"image\" style=\"float:center;width:{image_width//len(images)}%\">"
+                image_row += f"<div class=\"image\" style=\"float:{float};width:{image_width//len(images)}%\">"
                 image_row += f"<img src=\"data:image/png;base64,{data_uri}\" style=\"width:100%\">"
                 image_row += f"<center><div style=\"font-weight:bold;font-size:12;padding-bottom:20px\">{caption}</div></center>"
                 image_row += "</div>"
@@ -101,21 +106,32 @@ def add_paragraph(self, paragraph, font_size=14):
         """
         Parameters
         ----------
-        title: paragraph
+        paragraph: str
             The text to add to
             the report as a paragraph
         """
 
         self._body += f'<div style=\"font-size:{font_size}\"><p>{paragraph}</p></div>'
 
+    def add_line_breaks(self, num_breaks=1):
+        """
+        Parameters
+        ----------
+        num_breaks: paragraph
+            The text to add to
+            the report as a paragraph
+        """
+
+        for _ in range(num_breaks):
+            self._body += '<br>'
+
     def add_table(self, table):
         """
         Parameters
         ----------
         table: ResultTable
             The table we want to add
         """
-
         def table_style(border="1px solid black",
                         padding="5px 10px",
                         font_size="11pt",