Extend genai perf plots to compare across multiple runs (#635)

* Modify PlotManager and plots classes

* Support plots for multiple runs -draft

* Fix default plot visualization

* Remove artifact

* Set default compare directory

* Support generating parquet files

* Remove annotations and fix heatmap

* Fix errors

* Fix pre-commit

* Fix CodeQL warning

* Remove unused comments

* remove x axis tick label for boxplot

* Add logging and label for heatmap subplots

* Allow users to adjust width and height

* fix grammer

---------

Co-authored-by: Hyunjae Woo <[email protected]>

Author: 2 people

src/c++/perf_analyzer/genai-perf/genai_perf/constants.py

@@ -34,4 +34,5 @@
 
 
 DEFAULT_ARTIFACT_DIR = "artifacts"
+DEFAULT_COMPARE_DIR = "compare"
 DEFAULT_PARQUET_FILE = "all_data"

src/c++/perf_analyzer/genai-perf/genai_perf/logging.py

@@ -70,6 +70,16 @@ def init_logging() -> None:
                 "level": "DEBUG",
                 "propagate": False,
             },
+            "genai_perf.plots.plot_config_parser": {
+                "handlers": ["console"],
+                "level": "DEBUG",
+                "propagate": False,
+            },
+            "genai_perf.plots.plot_manager": {
+                "handlers": ["console"],
+                "level": "DEBUG",
+                "propagate": False,
+            },
         },
     }
     logging.config.dictConfig(LOGGING_CONFIG)

src/c++/perf_analyzer/genai-perf/genai_perf/main.py

@@ -118,10 +118,8 @@ def create_plots(filename: Path) -> None:
     PlotConfigParser.create_init_yaml_config([filename], output_dir)
     config_parser = PlotConfigParser(output_dir / "config.yaml")
     plot_configs = config_parser.generate_configs()
-
-    # TODO (harshini): plug-in configs to plot manager
-    # plot_manager = PlotManager(stats)
-    # plot_manager.create_default_plots()
+    plot_manager = PlotManager(plot_configs)
+    plot_manager.generate_plots()
 
 
 def finalize(profile_export_file: Path):

src/c++/perf_analyzer/genai-perf/genai_perf/parser.py

@@ -25,12 +25,13 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import argparse
+import os
 import sys
 from pathlib import Path
 
 import genai_perf.logging as logging
 import genai_perf.utils as utils
-from genai_perf.constants import CNN_DAILY_MAIL, OPEN_ORCA
+from genai_perf.constants import CNN_DAILY_MAIL, DEFAULT_COMPARE_DIR, OPEN_ORCA
 from genai_perf.llm_inputs.llm_inputs import LlmInputs, OutputFormat, PromptSource
 from genai_perf.plots.plot_config_parser import PlotConfigParser
 from genai_perf.plots.plot_manager import PlotManager
@@ -483,6 +484,11 @@ def _parse_compare_args(subparsers) -> argparse.ArgumentParser:
 ### Handlers ###
 
 
+def create_compare_dir() -> None:
+    if not os.path.exists(DEFAULT_COMPARE_DIR):
+        os.mkdir(DEFAULT_COMPARE_DIR)
+
+
 def profile_handler(args, extra_args):
     from genai_perf.wrapper import Profiler
 
@@ -492,15 +498,15 @@ def profile_handler(args, extra_args):
 def compare_handler(args: argparse.Namespace):
     """Handles `compare` subcommand workflow."""
     if args.files:
-        PlotConfigParser.create_init_yaml_config(args.files, Path("."))
-        args.config = Path("config.yaml")
+        create_compare_dir()
+        output_dir = Path(f"{DEFAULT_COMPARE_DIR}")
+        PlotConfigParser.create_init_yaml_config(args.files, output_dir)
+        args.config = output_dir / "config.yaml"
 
     config_parser = PlotConfigParser(args.config)
     plot_configs = config_parser.generate_configs()
-
-    # TODO (harshini): plug-in configs to PlotManager
-    # plot_manager = PlotManager(plot_configs)
-    # plot_manager.generate_plots()
+    plot_manager = PlotManager(plot_configs)
+    plot_manager.generate_plots()
 
 
 ### Entrypoint ###

src/c++/perf_analyzer/genai-perf/genai_perf/plots/base_plot.py

@@ -25,14 +25,11 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from pathlib import Path
 
-from copy import deepcopy
-from typing import Dict
-
-from genai_perf.constants import DEFAULT_ARTIFACT_DIR
+import pandas as pd
 from genai_perf.exceptions import GenAIPerfException
-from genai_perf.llm_metrics import Statistics
-from pandas import DataFrame
+from genai_perf.plots.plot_config import ProfileRunData
 from plotly.graph_objects import Figure
 
 
@@ -41,40 +38,44 @@ class BasePlot:
     Base class for plots
     """
 
-    def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
-        self._stats = stats
-        self._metrics_data = deepcopy(stats.metrics.data)
-        if extra_data:
-            self._metrics_data = self._metrics_data | extra_data
+    def __init__(self, data: list[ProfileRunData]) -> None:
+        self._profile_data = data
 
     def create_plot(
         self,
-        x_key: str,
-        y_key: str,
-        x_metric: str,
-        y_metric: str,
         graph_title: str,
         x_label: str,
         y_label: str,
+        width: int,
+        height: int,
         filename_root: str,
+        output_dir: Path,
     ) -> None:
         """
         Create plot for specific graph type
         """
         raise NotImplementedError
 
-    def _generate_parquet(self, dataframe: DataFrame, file: str) -> None:
-        dataframe.to_parquet(
-            f"{DEFAULT_ARTIFACT_DIR}/data/{file}.gzip", compression="gzip"
+    def _create_dataframe(self, x_label: str, y_label: str) -> pd.DataFrame:
+        return pd.DataFrame(
+            {
+                x_label: [prd.x_metric for prd in self._profile_data],
+                y_label: [prd.y_metric for prd in self._profile_data],
+                "Run Name": [prd.name for prd in self._profile_data],
+            }
         )
 
-    def _generate_graph_file(self, fig: Figure, file: str, title: str) -> None:
+    def _generate_parquet(self, df: pd.DataFrame, output_dir: Path, file: str) -> None:
+        filepath = output_dir / f"{file}.gzip"
+        df.to_parquet(filepath, compression="gzip")
+
+    def _generate_graph_file(self, fig: Figure, output_dir: Path, file: str) -> None:
         if file.endswith("jpeg"):
-            print(f"Generating '{title}' jpeg")
-            fig.write_image(f"{DEFAULT_ARTIFACT_DIR}/plots/{file}")
+            filepath = output_dir / f"{file}"
+            fig.write_image(filepath)
         elif file.endswith("html"):
-            print(f"Generating '{title}' html")
-            fig.write_html(f"{DEFAULT_ARTIFACT_DIR}/plots/{file}")
+            filepath = output_dir / f"{file}"
+            fig.write_html(filepath)
         else:
             extension = file.split(".")[-1]
             raise GenAIPerfException(f"image file type {extension} is not supported")

src/c++/perf_analyzer/genai-perf/genai_perf/plots/box_plot.py

@@ -25,105 +25,52 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from pathlib import Path
 
-import copy
-from typing import Dict
-
-import pandas as pd
-import plotly.express as px
-from genai_perf.llm_metrics import Statistics
+import plotly.graph_objects as go
 from genai_perf.plots.base_plot import BasePlot
-from genai_perf.utils import scale
-from plotly.graph_objects import Figure
+from genai_perf.plots.plot_config import ProfileRunData
 
 
 class BoxPlot(BasePlot):
     """
     Generate a box plot in jpeg and html format.
     """
 
-    def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
-        super().__init__(stats, extra_data)
+    def __init__(self, data: list[ProfileRunData]) -> None:
+        super().__init__(data)
 
     def create_plot(
         self,
-        x_key: str = "",
-        y_key: str = "",
-        x_metric: str = "",
-        y_metric: str = "",
         graph_title: str = "",
         x_label: str = "",
         y_label: str = "",
+        width: int = 700,
+        height: int = 450,
         filename_root: str = "",
+        output_dir: Path = Path(""),
     ) -> None:
-        df = pd.DataFrame({y_metric: self._metrics_data[y_key]})
-        fig = px.box(
-            df,
-            y=y_metric,
-            points="all",
-            title=graph_title,
+        fig = go.Figure()
+        for pd in self._profile_data:
+            fig.add_trace(go.Box(y=pd.y_metric, name=pd.name))
+
+        # Update layout and axis labels
+        fig.update_layout(
+            title={
+                "text": f"{graph_title}",
+                "xanchor": "center",
+                "x": 0.5,
+            },
+            width=width,
+            height=height,
         )
-        fig.update_layout(title_x=0.5)
-        fig.update_xaxes(title_text=x_label)
-
-        fig.update_yaxes(title_text="")
-
-        # create a copy to avoid annotations on html file
-        fig_jpeg = copy.deepcopy(fig)
-        self._add_annotations(fig_jpeg, y_metric)
-
-        self._generate_parquet(df, filename_root)
-        self._generate_graph_file(fig, filename_root + ".html", graph_title)
-        self._generate_graph_file(fig_jpeg, filename_root + ".jpeg", graph_title)
+        fig.update_traces(boxpoints="all")
+        fig.update_xaxes(title_text=x_label, showticklabels=False)
+        fig.update_yaxes(title_text=y_label)
 
-    def _add_annotations(self, fig: Figure, y_metric: str) -> None:
-        """
-        Add annotations to the non html version of the box plot
-        to replace the missing hovertext
-        """
-        stat_root_name = self._stats.metrics.get_base_name(y_metric)
+        # Save dataframe as parquet file
+        df = self._create_dataframe(x_label, y_label)
+        self._generate_parquet(df, output_dir, filename_root)
 
-        val = scale(self._stats.data[f"max_{stat_root_name}"], (1 / 1e9))
-        fig.add_annotation(
-            x=0.5,
-            y=val,
-            text=f"max: {round(val, 2)}",
-            showarrow=False,
-            yshift=10,
-        )
-
-        val = scale(self._stats.data[f"p75_{stat_root_name}"], (1 / 1e9))
-        fig.add_annotation(
-            x=0.5,
-            y=val,
-            text=f"q3: {round(val, 2)}",
-            showarrow=False,
-            yshift=10,
-        )
-
-        val = scale(self._stats.data[f"p50_{stat_root_name}"], (1 / 1e9))
-        fig.add_annotation(
-            x=0.5,
-            y=val,
-            text=f"median: {round(val, 2)}",
-            showarrow=False,
-            yshift=10,
-        )
-
-        val = scale(self._stats.data[f"p25_{stat_root_name}"], (1 / 1e9))
-        fig.add_annotation(
-            x=0.5,
-            y=val,
-            text=f"q1: {round(val, 2)}",
-            showarrow=False,
-            yshift=10,
-        )
-
-        val = scale(self._stats.data[f"min_{stat_root_name}"], (1 / 1e9))
-        fig.add_annotation(
-            x=0.5,
-            y=val,
-            text=f"min: {round(val, 2)}",
-            showarrow=False,
-            yshift=10,
-        )
+        self._generate_graph_file(fig, output_dir, filename_root + ".html")
+        self._generate_graph_file(fig, output_dir, filename_root + ".jpeg")