Fix compiler bug (#7482)

1. fix compiler name bug for geomean, it should be geomean_speedup
2. filter export for all other aggregated metrics except passrate

https://torchci-git-fixcompilerbug-fbopensource.vercel.app/benchmark/compilers_regression
3. render missing data with light grey if it's enabled

Author: yangw-dev

aws/lambda/benchmark_regression_summary_report/common/config.py

@@ -228,8 +228,8 @@
                 threshold=0.9,
                 baseline_aggregation="max",
             ),
-            "geomean": RegressionPolicy(
-                name="geomean",
+            "geomean_speedup": RegressionPolicy(
+                name="geomean_speedup",
                 condition="greater_equal",
                 threshold=0.95,
                 baseline_aggregation="max",

torchci/components/benchmark_v3/components/dataRender/components/benchmarkTimeSeries/components/BenchmarkTimeSeriesComparisonSection/BenchmarkTimeSeriesComparisonTable/ComparisonTableColumnRendering.tsx

@@ -157,6 +157,7 @@ export function getComparisionTableConlumnRendering(
 const VIOLATE_RULE_COLOR = "#ffebee"; // red[50]
 const IMPROVEMENT_COLOR = "#e8f5e9"; // green[50]
 const WARNING_COLOR = "#fff9c4"; // yellow[50]
+const MISSING_DATA_COLOR = "#F5F5F5"; // ~ MUI grey[300]
 
 export function ComparisonTablePrimaryFieldValueCell({
   params,
@@ -222,6 +223,9 @@ export function ComparisonTableColumnFieldValueCell({
     case "warning":
       bgColor = WARNING_COLOR;
       break;
+    case "missing":
+      bgColor = MISSING_DATA_COLOR;
+      break;
     case "neutral":
     default:
       break;
@@ -365,6 +369,21 @@ export function getComparisonResult(
       ? missingText
       : config?.renderOptions?.bothMissingText;
 
+  // if either side missing, mark as missing
+  if (config?.renderOptions?.renderMissing) {
+    if (ldata == null && rdata == null) {
+      result.verdict = "missing";
+      result.reason = "both missing";
+    } else if (ldata == null) {
+      result.verdict = "missing";
+      result.reason = "left missing";
+    } else if (rdata == null) {
+      result.verdict = "missing";
+      result.reason = "right missing";
+    }
+  }
+
+  // if either side failed, mark as failure, failure is higher priority than missing
   if (ldata?.is_failure || rdata?.is_failure) {
     result.verdict = "warning";
     result.reason = "detect failure";
@@ -382,6 +401,7 @@ export function getComparisonResult(
     missingText,
     bothMissingText
   );
+
   return {
     result,
     text,

torchci/components/benchmark_v3/components/dataRender/components/benchmarkTimeSeries/helper.tsx

@@ -68,6 +68,7 @@ export interface BenchmarkComparisonTableRenderingOptions {
   };
   missingText?: string;
   bothMissingText?: string;
+  renderMissing?: boolean;
 }
 
 export interface BenchmarkComparisonTablePrimaryColumnConfig {

torchci/components/benchmark_v3/configs/helpers/RegressionPolicy.ts

@@ -1,6 +1,11 @@
 import { asNumber } from "components/benchmark_v3/components/dataRender/components/benchmarkTimeSeries/components/BenchmarkTimeSeriesComparisonSection/BenchmarkTimeSeriesComparisonTable/ComparisonTableHelpers";
 
-export type ComparisonVerdict = "good" | "neutral" | "regression" | "warning";
+export type ComparisonVerdict =
+  | "good"
+  | "neutral"
+  | "regression"
+  | "warning"
+  | "missing";
 export type ComparisonPolicyType = "ratio" | "status" | "threshold";
 export const DEFAULT_TYPE = "ratio";
 export const DEFAULT_BAD_RATIO = 0.9;

torchci/components/benchmark_v3/configs/teams/compilers/CompilerPrecomputeConfirmDialogContent.tsx

@@ -62,10 +62,9 @@ export const CompilerPrecomputeConfirmDialogContent: React.FC<
       triggerUpdate();
       return;
     }
-
     const cell = await navigateToDataGrid(
       tableId,
-      [`${left?.compiler}`],
+      [`${left?.compiler}|`],
       `${left?.suite}`,
       toggleSectonId
     );

torchci/components/benchmark_v3/configs/teams/compilers/config.ts

@@ -32,7 +32,7 @@ const PASSRATE_COMPARISON_POLICY: BenchmarkComparisonPolicyConfig = {
   },
 };
 const GEOMEAN_COMPARISON_POLICY: BenchmarkComparisonPolicyConfig = {
-  target: "geomean",
+  target: "geomean_speedup",
   type: "ratio",
   ratioPolicy: {
     badRatio: 0.95,
@@ -107,7 +107,7 @@ const RENDER_MAPPING_BOOK = {
       scale: 100,
     },
   },
-  geomean: {
+  geomean_speedup: {
     unit: {
       unit: "x",
     },
@@ -323,6 +323,7 @@ export const CompilerDashboardBenchmarkUIConfig: BenchmarkUIConfig = {
           extraMetadata: DASHBOARD_COMPARISON_TABLE_METADATA_COLUMNS,
           renderOptions: {
             tableRenderingBook: DashboardRenderBook,
+            renderMissing: true,
             flex: {
               primary: 2,
             },
@@ -418,7 +419,7 @@ export const CompilerPrecomputeBenchmarkUIConfig: BenchmarkUIConfig = {
                   passrate: {
                     text: "Passrate",
                   },
-                  geomean: {
+                  geomean_speedup: {
                     text: "Geometric mean speedup",
                   },
                   compilation_latency: {
@@ -447,7 +448,7 @@ export const CompilerPrecomputeBenchmarkUIConfig: BenchmarkUIConfig = {
           filterByFieldValues: {
             metric: [
               "passrate",
-              "geomean",
+              "geomean_speedup",
               "compilation_latency",
               "compression_ratio",
             ],
@@ -465,7 +466,7 @@ export const CompilerPrecomputeBenchmarkUIConfig: BenchmarkUIConfig = {
             targetField: "metric",
             comparisonPolicy: {
               passrate: PASSRATE_COMPARISON_POLICY,
-              geomean: GEOMEAN_COMPARISON_POLICY,
+              geomean_speedup: GEOMEAN_COMPARISON_POLICY,
               compilation_latency: COMPILATION_LATENCY_COMPARISON_POLICY,
               compression_ratio: COMPRESSION_RATIO_POLICY,
             },
@@ -474,17 +475,18 @@ export const CompilerPrecomputeBenchmarkUIConfig: BenchmarkUIConfig = {
                 passrate: {
                   text: "Passrate (threshold: 95%)",
                 },
-                geomean: {
+                geomean_speedup: {
                   text: "Geometric mean speedup (threshold = 0.95x)",
                 },
                 compilation_latency: {
-                  text: "compilation time (seconds)",
+                  text: "Compilation time (seconds)",
                 },
                 compression_ratio: {
                   text: "Peak memory footprint compression ratio (threshold = 0.95x)",
                 },
               },
               tableRenderingBook: RENDER_MAPPING_BOOK,
+              renderMissing: true,
             },
           },
         },

torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts

@@ -67,6 +67,8 @@ export function toPrecomputeCompilerData(
   const executionTime = computeExecutionTime(data, models);
   const peakMemoryUsage = computePeakMemoryUsage(data, models);
 
+  // filter out export for compiler since it's always 0
+
   let all_data = [
     passrate,
     geomean,
@@ -76,6 +78,10 @@ export function toPrecomputeCompilerData(
     peakMemoryUsage,
   ].flat();
 
+  // only show export for passrate
+  all_data = all_data.filter((row) =>
+    row.compiler == "export" && row.metric != "passrate" ? false : true
+  );
   all_data = [...all_data].sort(
     (a, b) =>
       Date.parse(a.granularity_bucket) - Date.parse(b.granularity_bucket)

torchci/lib/benchmark/compilerUtils.ts

@@ -166,7 +166,7 @@ export function computeGeomean(
 
     const [bucket, workflowId, suite, compiler] = key.split("+");
     returnedGeomean.push({
-      metric: "geomean",
+      metric: "geomean_speedup",
       value: Number(gm),
       granularity_bucket: bucket,
       workflow_id: workflowId,