Skip to content

Commit f0936f4

Browse files
debermudezdebermudez
authored
add visualizations (#558)
* Initial ttft box plot implementation * Add request latency chart * Add heatmap graph * Add scatter plot graph * Add base class for plots * Update requirements * Add plot manager and refactor to use a base class * Add parquet output to genai-perf * Use new plot manager class and update gitignore to ignore artifacts directory * Add compression to parquet files * Remove unused imports * Fix annotations for box plots * Rename labels and update another feedback * Fix title and formatting * Add comment for deep copy * Update plots to preprocess data and use a common API * Uncomment run method and error logging * Remove * in arg list for plots * Disable failing test: test_random_synthetic
1 parent 877f16e commit f0936f4

File tree

12 files changed

+592
-31
lines changed

12 files changed

+592
-31
lines changed

‎src/c++/perf_analyzer/genai-perf/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
artifacts/

‎src/c++/perf_analyzer/genai-perf/genai_perf/constants.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,7 @@
3333
OPEN_ORCA = "openorca"
3434
CNN_DAILY_MAIL = "cnn_dailymail"
3535
DEFAULT_INPUT_DATA_JSON = "llm_inputs.json"
36+
37+
38+
DEFAULT_ARTIFACT_DIR = "artifacts"
39+
DEFAULT_PARQUET_FILE = "all_data"

‎src/c++/perf_analyzer/genai-perf/genai_perf/graphs/base_plot.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
29+
from copy import deepcopy
30+
from typing import Dict
31+
32+
from genai_perf.constants import DEFAULT_ARTIFACT_DIR
33+
from genai_perf.exceptions import GenAIPerfException
34+
from genai_perf.llm_metrics import Statistics
35+
from pandas import DataFrame
36+
from plotly.graph_objects import Figure
37+
38+
39+
class BasePlot:
40+
"""
41+
Base class for plots
42+
"""
43+
44+
def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
45+
self._stats = stats
46+
self._metrics_data = deepcopy(stats.metrics.data)
47+
if extra_data:
48+
self._metrics_data = self._metrics_data | extra_data
49+
50+
def create_plot(
51+
self,
52+
x_key: str,
53+
y_key: str,
54+
x_metric: str,
55+
y_metric: str,
56+
graph_title: str,
57+
x_label: str,
58+
y_label: str,
59+
filename_root: str,
60+
) -> None:
61+
"""
62+
Create plot for specific graph type
63+
"""
64+
raise NotImplementedError
65+
66+
def _generate_parquet(self, dataframe: DataFrame, file: str):
67+
dataframe.to_parquet(
68+
f"{DEFAULT_ARTIFACT_DIR}/data/{file}.gzip", compression="gzip"
69+
)
70+
71+
def _generate_graph_file(self, fig: Figure, file: str, title: str):
72+
if file.endswith("jpeg"):
73+
print(f"Generating '{title}' jpeg")
74+
fig.write_image(f"{DEFAULT_ARTIFACT_DIR}/images/{file}")
75+
elif file.endswith("html"):
76+
print(f"Generating '{title}' html")
77+
fig.write_html(f"{DEFAULT_ARTIFACT_DIR}/images/{file}")
78+
else:
79+
extension = file.split(".")[-1]
80+
raise GenAIPerfException(f"image file type {extension} is not supported")

‎src/c++/perf_analyzer/genai-perf/genai_perf/graphs/box_plot.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
29+
import copy
30+
from typing import Dict
31+
32+
import pandas as pd
33+
import plotly.express as px
34+
from genai_perf.graphs.base_plot import BasePlot
35+
from genai_perf.llm_metrics import Statistics
36+
from genai_perf.utils import scale
37+
from plotly.graph_objects import Figure
38+
39+
40+
class BoxPlot(BasePlot):
41+
"""
42+
Generate a box plot in jpeg and html format.
43+
"""
44+
45+
def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
46+
super().__init__(stats, extra_data)
47+
48+
def create_plot(
49+
self,
50+
x_key: str = "",
51+
y_key: str = "",
52+
x_metric: str = "",
53+
y_metric: str = "",
54+
graph_title: str = "",
55+
x_label: str = "",
56+
y_label: str = "",
57+
filename_root: str = "",
58+
):
59+
df = pd.DataFrame({y_metric: self._metrics_data[y_key]})
60+
fig = px.box(
61+
df,
62+
y=y_metric,
63+
points="all",
64+
title=graph_title,
65+
)
66+
fig.update_layout(title_x=0.5)
67+
fig.update_xaxes(title_text=x_label)
68+
69+
fig.update_yaxes(title_text="")
70+
71+
# create a copy to avoid annotations on html file
72+
fig_jpeg = copy.deepcopy(fig)
73+
self._add_annotations(fig_jpeg, y_metric)
74+
75+
self._generate_parquet(df, filename_root)
76+
self._generate_graph_file(fig, filename_root + ".html", graph_title)
77+
self._generate_graph_file(fig_jpeg, filename_root + ".jpeg", graph_title)
78+
79+
def _add_annotations(self, fig: Figure, y_metric: str) -> None:
80+
"""
81+
Add annotations to the non html version of the box plot
82+
to replace the missing hovertext
83+
"""
84+
stat_root_name = self._stats.metrics.get_base_name(y_metric)
85+
86+
val = scale(self._stats.data[f"max_{stat_root_name}"], (1 / 1e9))
87+
fig.add_annotation(
88+
x=0.5,
89+
y=val,
90+
text=f"max: {round(val, 2)}",
91+
showarrow=False,
92+
yshift=10,
93+
)
94+
95+
val = scale(self._stats.data[f"p75_{stat_root_name}"], (1 / 1e9))
96+
fig.add_annotation(
97+
x=0.5,
98+
y=val,
99+
text=f"q3: {round(val, 2)}",
100+
showarrow=False,
101+
yshift=10,
102+
)
103+
104+
val = scale(self._stats.data[f"p50_{stat_root_name}"], (1 / 1e9))
105+
fig.add_annotation(
106+
x=0.5,
107+
y=val,
108+
text=f"median: {round(val, 2)}",
109+
showarrow=False,
110+
yshift=10,
111+
)
112+
113+
val = scale(self._stats.data[f"p25_{stat_root_name}"], (1 / 1e9))
114+
fig.add_annotation(
115+
x=0.5,
116+
y=val,
117+
text=f"q1: {round(val, 2)}",
118+
showarrow=False,
119+
yshift=10,
120+
)
121+
122+
val = scale(self._stats.data[f"min_{stat_root_name}"], (1 / 1e9))
123+
fig.add_annotation(
124+
x=0.5,
125+
y=val,
126+
text=f"min: {round(val, 2)}",
127+
showarrow=False,
128+
yshift=10,
129+
)

‎src/c++/perf_analyzer/genai-perf/genai_perf/graphs/heat_map.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
from typing import Dict
29+
30+
import pandas as pd
31+
import plotly.express as px
32+
from genai_perf.graphs.base_plot import BasePlot
33+
from genai_perf.llm_metrics import Statistics
34+
35+
36+
class HeatMap(BasePlot):
37+
"""
38+
Generate a heat map in jpeg and html format.
39+
"""
40+
41+
def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
42+
super().__init__(stats, extra_data)
43+
44+
def create_plot(
45+
self,
46+
x_key: str = "",
47+
y_key: str = "",
48+
x_metric: str = "",
49+
y_metric: str = "",
50+
graph_title: str = "",
51+
x_label: str = "",
52+
y_label: str = "",
53+
filename_root: str = "",
54+
):
55+
x_values = self._metrics_data[x_key]
56+
y_values = self._metrics_data[y_key]
57+
df = pd.DataFrame(
58+
{
59+
x_metric: x_values,
60+
y_metric: y_values,
61+
}
62+
)
63+
fig = px.density_heatmap(
64+
df,
65+
x=x_metric,
66+
y=y_metric,
67+
)
68+
fig.update_layout(
69+
title={
70+
"text": graph_title,
71+
"xanchor": "center",
72+
"x": 0.5,
73+
}
74+
)
75+
fig.update_xaxes(title_text=x_label)
76+
fig.update_yaxes(title_text=y_label)
77+
78+
self._generate_parquet(df, filename_root)
79+
self._generate_graph_file(fig, filename_root + ".html", graph_title)
80+
self._generate_graph_file(fig, filename_root + ".jpeg", graph_title)

0 commit comments

Comments
 (0)