Add TorchAO LLM benchmark dashboard (#6087)

A simple change to add the initial version of TorchAO LLM benchmark
dashboard.

### Testing


https://torchci-git-fork-huydhn-add-ao-llm-dashboard-fbopensource.vercel.app/benchmark/llms?repoName=pytorch%2Fao

Author: huydhn

torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json

@@ -4,6 +4,7 @@
   "excludedMetrics": "Array(String)",
   "benchmarks": "Array(String)",
   "models": "Array(String)",
+  "backends": "Array(String)",
   "repo": "String",
   "startTime": "DateTime64(3)",
   "stopTime": "DateTime64(3)"

torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql

@@ -30,6 +30,10 @@ WITH benchmarks AS (
             has({models: Array(String) }, o.model.name)
             OR empty({models: Array(String) })
         )
+        AND (
+            has({backends: Array(String) }, o.model.backend)
+            OR empty({backends: Array(String) })
+        )
         AND (
             has({dtypes: Array(String) }, o.benchmark.dtype)
             OR empty({dtypes: Array(String) })

torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json

@@ -7,6 +7,7 @@
   "benchmarks": "Array(String)",
   "granularity": "String",
   "models": "Array(String)",
+  "backends": "Array(String)",
   "repo": "String",
   "startTime": "DateTime64(3)",
   "stopTime": "DateTime64(3)"

torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql

@@ -44,6 +44,10 @@ WITH benchmarks AS (
             has({models: Array(String) }, o.model.name)
             OR empty({models: Array(String) })
         )
+        AND (
+            has({backends: Array(String) }, o.model.backend)
+            OR empty({backends: Array(String) })
+        )
         AND (
             has({dtypes: Array(String) }, o.benchmark.dtype)
             OR empty({dtypes: Array(String) })
@@ -58,7 +62,8 @@ WITH benchmarks AS (
 SELECT
     DISTINCT workflow_id,
     job_id,
-    CONCAT(model, ' ', backend) AS name,
+    model,
+    backend,
     metric,
     actual,
     target,
@@ -70,10 +75,6 @@ FROM
     benchmarks
 WHERE
     (
-        has({models: Array(String) }, CONCAT(model, ' ', backend))
-        OR empty({models: Array(String) })
-    )
-    AND (
         has({branches: Array(String) }, head_branch)
         OR empty({branches: Array(String) })
     )
@@ -91,6 +92,7 @@ WHERE
 ORDER BY
     granularity_bucket DESC,
     workflow_id DESC,
-    name,
+    backend,
+    model,
     dtype,
     device

torchci/clickhouse_queries/oss_ci_benchmark_names/params.json

@@ -4,6 +4,7 @@
   "excludedMetrics": "Array(String)",
   "benchmarks": "Array(String)",
   "models": "Array(String)",
+  "backends": "Array(String)",
   "repo": "String",
   "startTime": "DateTime64(3)",
   "stopTime": "DateTime64(3)"

torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql

@@ -30,6 +30,10 @@ WITH benchmarks AS (
             has({models: Array(String) }, o.model.name)
             OR empty({models: Array(String) })
         )
+        AND (
+            has({backends: Array(String) }, o.model.backend)
+            OR empty({backends: Array(String) })
+        )
         AND (
             has({dtypes: Array(String) }, o.benchmark.dtype)
             OR empty({dtypes: Array(String) })
@@ -43,7 +47,8 @@ WITH benchmarks AS (
 )
 SELECT
     DISTINCT benchmark,
-    CONCAT(model, ' ', backend) AS name,
+    model,
+    backend,
     metric,
     dtype,
     device,
@@ -64,7 +69,8 @@ WHERE
     AND notEmpty(device)
 ORDER BY
     benchmark,
-    name,
+    backend,
+    model,
     metric,
     dtype,
     device

torchci/components/NavBar.tsx

@@ -50,13 +50,17 @@ function NavBar() {
       href: "/torchbench/userbenchmark",
     },
     {
-      name: "LLMs",
+      name: "PyTorch LLMs",
       href: "/benchmark/llms?repoName=pytorch%2Fpytorch",
     },
     {
       name: "ExecuTorch",
       href: "/benchmark/llms?repoName=pytorch%2Fexecutorch",
     },
+    {
+      name: "TorchAO LLMs",
+      href: "/benchmark/llms?repoName=pytorch%2Fao",
+    },
   ];
 
   const devInfraDropdown = [

torchci/components/benchmark/llms/ModelGraphPanel.tsx

@@ -27,6 +27,7 @@ export function GraphPanel({
   queryParams,
   granularity,
   modelName,
+  backendName,
   dtypeName,
   deviceName,
   metricNames,
@@ -36,6 +37,7 @@ export function GraphPanel({
   queryParams: { [key: string]: any };
   granularity: Granularity;
   modelName: string;
+  backendName: string;
   dtypeName: string;
   deviceName: string;
   metricNames: string[];
@@ -44,16 +46,10 @@ export function GraphPanel({
 }) {
   // Do not set the commit here to query all the records in the time range to
   // draw a chart
-  const { data, error } = useBenchmark(
-    queryParams,
-    modelName,
-    dtypeName,
-    deviceName,
-    {
-      branch: rBranchAndCommit.branch,
-      commit: "",
-    }
-  );
+  const { data, error } = useBenchmark(queryParams, {
+    branch: rBranchAndCommit.branch,
+    commit: "",
+  });
 
   if (data === undefined || data.length === 0) {
     return (
@@ -86,7 +82,7 @@ export function GraphPanel({
     chartData[metric] = data
       .filter((record: LLMsBenchmarkData) => {
         return (
-          record.name === modelName &&
+          record.model === modelName &&
           (record.dtype === dtypeName || dtypeName === DEFAULT_DTYPE_NAME) &&
           (`${record.device} (${record.arch})` === deviceName ||
             deviceName === DEFAULT_DEVICE_NAME) &&
@@ -102,17 +98,17 @@ export function GraphPanel({
         );
       })
       .map((record: LLMsBenchmarkData) => {
-        const name = record.name;
+        const model = record.model;
         const dtype = record.dtype;
         const device = record.device;
 
-        record.display = name.includes(dtype)
-          ? name.includes(device)
-            ? name
-            : `${name} (${device})`
-          : name.includes(device)
-          ? `${name} (${dtype})`
-          : `${name} (${dtype} / ${device})`;
+        record.display = model.includes(dtype)
+          ? model.includes(device)
+            ? model
+            : `${model} (${device})`
+          : model.includes(device)
+          ? `${model} (${dtype})`
+          : `${model} (${dtype} / ${device})`;
 
         return record;
       });