[CI][Benchmarks] Update of benchmark dashboard - flamegraphs

devops/actions/run-tests/benchmark/action.yml

@@ -133,6 +133,11 @@ runs:
       cmake --install build
 
       cd -
+  - name: Install linux-tools package
+    shell: bash
+    run: |
+      sudo apt-get update
+      sudo apt-get install -y linux-tools-generic linux-tools-$(uname -r)
   - name: Checkout results repo
     shell: bash
     run: |
@@ -203,7 +208,8 @@ runs:
         --output-dir "./llvm-ci-perf-results/" \
         --preset "$PRESET" \
         --timestamp-override "$SAVE_TIMESTAMP" \
-        --detect-version sycl,compute_runtime
+        --detect-version sycl,compute_runtime \
+        --flamegraph inclusive
 
       echo "-----"
       python3 ./devops/scripts/benchmarks/compare.py to_hist \

devops/scripts/benchmarks/html/styles.css

@@ -206,6 +206,39 @@ details[open] summary::after {
 .download-button:hover {
     color: var(--color-cyan);
 }
+
+.download-button:disabled {
+    color: var(--text-muted);
+    cursor: not-allowed;
+}
+
+.download-list {
+    position: absolute !important;
+    z-index: 1000 !important;
+    background: var(--bg-white) !important;
+    border: 1px solid var(--border-medium) !important;
+    border-radius: 4px !important;
+    box-shadow: var(--shadow-dropdown) !important;
+    padding: 4px !important;
+    min-width: 200px !important;
+    max-width: 300px !important;
+}
+
+.download-list a {
+    display: block !important;
+    padding: 8px 12px !important;
+    text-decoration: none !important;
+    color: var(--text-dark) !important;
+    border-radius: 2px !important;
+    font-size: 14px !important;
+    white-space: nowrap !important;
+    overflow: hidden !important;
+    text-overflow: ellipsis !important;
+}
+
+.download-list a:hover {
+    background-color: var(--bg-light) !important;
+}
 .loading-indicator {
     text-align: center;
     font-size: 18px;
@@ -369,6 +402,48 @@ details[open] summary::after {
     cursor: help;
     font-size: 12px;
 }
+
+/* Flamegraph link area styles (used by scripts.js) */
+.chart-flamegraph-links {
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    margin-top: 8px;
+    /* small margin below links to separate from any gray bar or footer */
+    margin-bottom: 6px;
+}
+.flamegraph-label {
+    color: var(--text-dark);
+    font-weight: 600;
+}
+.flamegraph-links-inline {
+    display: flex;
+    gap: 8px;
+    flex-wrap: wrap;
+}
+.flamegraph-link {
+    color: var(--text-warning);
+    text-decoration: none;
+    font-weight: 500;
+    display: inline-flex;
+    align-items: center;
+    gap: 6px;
+}
+.flamegraph-link:hover {
+    text-decoration: underline;
+}
+.flame-icon {
+    font-size: 16px;
+    line-height: 1;
+    display: inline-block;
+}
+.flame-text {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    max-width: 220px;
+    display: inline-block;
+}
 #tag-filters {
     display: flex;
     flex-wrap: wrap;
@@ -530,30 +605,63 @@ details[open] summary::after {
     border: 1px solid var(--border-medium);
     border-radius: 4px;
     display: block;
-    margin: 10px auto;
+    margin: 0 0 10px 0;
     transition: all 0.3s ease;
     box-sizing: border-box;
     overflow: hidden;
+    /* Ensure maximum width utilization */
+    max-width: none;
+    min-width: 0;
+}
+
+/* Flamegraph container styles - gives each flamegraph its own space */
+.flamegraph-container {
+    margin-bottom: 20px;
+    padding: 0;
+    border: none;
+    border-radius: 0;
+    background-color: transparent;
+    width: 100%;
+    /* Ensure no width constraints */
+    max-width: none;
+    min-width: 0;
+    box-sizing: border-box;
+}
+
+.flamegraph-container:last-child {
+    margin-bottom: 0;
 }
 
-.flamegraph-iframe:first-child {
-    margin: 0 auto 10px auto;
+.flamegraph-title {
+    font-size: 16px;
+    font-weight: 600;
+    color: var(--text-dark);
+    margin: 0 0 10px 0;
+    padding: 8px 12px;
+    background-color: var(--bg-light);
+    border-radius: 4px;
+    border-left: 4px solid var(--color-blue);
 }
 
 /* Ensure flamegraph containers have proper spacing and fit within container */
-.chart-container iframe {
-    margin-bottom: 10px;
+.chart-container .flamegraph-container {
+    margin-bottom: 20px;
+    width: 100%;
 }
 
-/* Handle multiple flamegraphs displayed vertically */
-.chart-content iframe[src*="flamegraphs"]:not(:last-child) {
-    margin-bottom: 15px;
-    border-bottom: 2px solid var(--bg-light);
+/* Reduce padding for chart containers that contain flamegraphs to maximize space */
+.chart-container.flamegraph-chart {
+    padding: 8px;
 }
 
-/* Add subtle visual separation between multiple flamegraphs */
-.chart-content iframe[src*="flamegraphs"]:not(:first-child) {
-    margin-top: 15px;
+/* Ensure chart content doesn't constrain flamegraphs */
+.chart-container.flamegraph-chart .chart-content {
+    padding: 0;
+}
+
+/* Handle multiple flamegraphs displayed vertically - now handled by container */
+.flamegraph-container:not(:last-child) {
+    margin-bottom: 20px;
 }
 
 /* Floating flamegraph download list */

devops/scripts/benchmarks/output_html.py

@@ -71,50 +71,34 @@ def _write_output_to_file(
     # Define variable configuration based on whether we're archiving or not
     filename = "data_archive" if archive else "data"
 
+    # Emit unified canonical format for both local (JS wrapper) and remote (pure JSON)
+    output_data = json.loads(output.to_json())  # type: ignore
+    output_data["defaultCompareNames"] = output.default_compare_names
+
+    if options.flamegraph:
+        flamegraph_data = _get_flamegraph_data(html_path)
+        if flamegraph_data and flamegraph_data.get("runs"):
+            output_data["flamegraphData"] = flamegraph_data
+            log.debug(
+                f"Added flamegraph data for {len(flamegraph_data['runs'])} runs to {filename}.*"
+            )
+
     if options.output_html == "local":
+        # Single JS assignment for canonical object
         data_path = os.path.join(html_path, f"{filename}.js")
         with open(data_path, "w") as f:
-            # For local format, we need to write JavaScript variable assignments
+            # kept for backward references, but embed full object under runs
             f.write("benchmarkRuns = ")
-            json.dump(json.loads(output.to_json())["runs"], f, indent=2)  # type: ignore
-            f.write(";\n\n")
-
-            f.write(f"benchmarkMetadata = ")
-            json.dump(json.loads(output.to_json())["metadata"], f, indent=2)  # type: ignore
-            f.write(";\n\n")
-
-            f.write(f"benchmarkTags = ")
-            json.dump(json.loads(output.to_json())["tags"], f, indent=2)  # type: ignore
-            f.write(";\n\n")
-
-            f.write(f"defaultCompareNames = ")
-            json.dump(output.default_compare_names, f, indent=2)
-            f.write(";\n\n")
-
-            # Add flamegraph data if it exists
-            if options.flamegraph:
-                flamegraph_data = _get_flamegraph_data(html_path)
-                if flamegraph_data and flamegraph_data.get("runs"):
-                    f.write("flamegraphData = ")
-                    json.dump(flamegraph_data, f, indent=2)
-                    f.write(";\n\n")
-                    log.debug(
-                        f"Added flamegraph data for {len(flamegraph_data['runs'])} runs to data.js"
-                    )
-
-            if not archive:
-                log.info(f"See {html_path}/index.html for the results.")
+            f.write(
+                "undefined; /* placeholder to preserve legacy global; use benchmarkDataCanonical instead */\n"
+            )
+            f.write("benchmarkDataCanonical = ")
+            json.dump(output_data, f, indent=2)
+            f.write(";\n")
+        if not archive:
+            log.info(f"See {html_path}/index.html for the results.")
     else:
-        # For remote format, we write a single JSON file
         data_path = os.path.join(html_path, f"{filename}.json")
-        output_data = json.loads(output.to_json())  # type: ignore
-        if options.flamegraph:
-            flamegraph_data = _get_flamegraph_data(html_path)
-            if flamegraph_data and flamegraph_data.get("runs"):
-                output_data["flamegraphs"] = flamegraph_data
-                log.debug(
-                    f"Added flamegraph data for {len(flamegraph_data['runs'])} runs to {filename}.json"
-                )
         with open(data_path, "w") as f:
             json.dump(output_data, f, indent=2)
         log.info(

devops/scripts/benchmarks/utils/flamegraph.py

@@ -8,7 +8,13 @@
 from pathlib import Path
 
 from options import options
-from utils.utils import run, git_clone, prune_old_files, remove_by_prefix
+from utils.utils import (
+    run,
+    git_clone,
+    prune_old_files,
+    remove_by_prefix,
+    sanitize_filename,
+)
 from utils.logger import log
 
 from datetime import datetime, timezone
@@ -40,7 +46,7 @@ def __init__(self):
 
         if options.results_directory_override:
             self.flamegraphs_dir = (
-                Path(options.results_directory_override) / "flamegraphs"
+                Path(options.results_directory_override) / "results" / "flamegraphs"
             )
         else:
             self.flamegraphs_dir = Path(options.workdir) / "results" / "flamegraphs"
@@ -67,7 +73,9 @@ def setup(
                 "perf command not found. Please install linux-tools or perf package."
             )
 
-        dir_name = f"{suite_name}__{bench_name}"
+        sanitized_suite_name = sanitize_filename(suite_name)
+        sanitized_bench_name = sanitize_filename(bench_name)
+        dir_name = f"{sanitized_suite_name}__{sanitized_bench_name}"
         bench_dir = self.flamegraphs_dir / dir_name
         bench_dir.mkdir(parents=True, exist_ok=True)
 
@@ -106,13 +114,18 @@ def handle_output(self, bench_name: str, perf_data_file: str, suite_name: str =
             perf_data_path.stem.replace(".perf", "") + ".svg"
         )
         folded_file = perf_data_path.with_suffix(".folded")
-
         try:
             self._convert_perf_to_folded(perf_data_path, folded_file)
             self._generate_svg(folded_file, svg_file, bench_name)
 
-            log.debug(f"Generated flamegraph: {svg_file}")
+            log.info(f"FlameGraph SVG created: {svg_file.resolve()}")
             self._create_immediate_symlink(svg_file)
+
+            # Clean up the original perf data file after successful SVG generation
+            if perf_data_path.exists():
+                perf_data_path.unlink()
+                log.debug(f"Removed original perf data file: {perf_data_path}")
+
             prune_old_files(str(perf_data_path.parent))
             return str(svg_file)
         except Exception as e:

devops/scripts/benchmarks/utils/unitrace.py

@@ -12,6 +12,7 @@
     prune_old_files,
     remove_by_prefix,
     remove_by_extension,
+    sanitize_filename,
 )
 from utils.logger import log
 
@@ -65,7 +66,9 @@ def __init__(self):
         if options.results_directory_override == None:
             self.traces_dir = os.path.join(options.workdir, "results", "traces")
         else:
-            self.traces_dir = os.path.join(options.results_directory_override, "traces")
+            self.traces_dir = os.path.join(
+                options.results_directory_override, "results", "traces"
+            )
 
     def _prune_unitrace_dirs(self, res_dir: str, FILECNT: int = 10):
         """Keep only the last FILECNT files in the traces directory."""
@@ -91,7 +94,8 @@ def setup(
         if not os.path.exists(unitrace_bin):
             raise FileNotFoundError(f"Unitrace binary not found: {unitrace_bin}. ")
         os.makedirs(self.traces_dir, exist_ok=True)
-        bench_dir = os.path.join(f"{self.traces_dir}", f"{bench_name}")
+        sanitized_bench_name = sanitize_filename(bench_name)
+        bench_dir = os.path.join(f"{self.traces_dir}", f"{sanitized_bench_name}")
 
         os.makedirs(bench_dir, exist_ok=True)
 
@@ -162,6 +166,8 @@ def handle_output(self, unitrace_output: str):
         shutil.move(os.path.join(options.benchmark_cwd, pid_json_files[-1]), json_name)
         log.debug(f"Moved {pid_json_files[-1]} to {json_name}")
 
+        log.info(f"Unitrace output files: {unitrace_output}, {json_name}")
+
         # Prune old unitrace directories
         self._prune_unitrace_dirs(os.path.dirname(unitrace_output))
 

devops/scripts/benchmarks/utils/utils.py

@@ -19,6 +19,18 @@
 from utils.logger import log
 
 
+def sanitize_filename(name: str) -> str:
+    """
+    Sanitize a string to be safe for use as a filename or directory name.
+    Replace invalid characters with underscores.
+    Invalid characters: " : < > | * ? \r \n
+    """
+    # Replace invalid characters with underscores
+    invalid_chars = r'[":;<>|*?\r\n]'
+    sanitized = re.sub(invalid_chars, "_", name)
+    return sanitized
+
+
 def run(
     command,
     env_vars={},