final

Signed-off-by: Yang Wang <[email protected]>

Author: yangw-dev

.ci/scripts/benchmark_tooling/execu_torch_read_data.py

@@ -0,0 +1,75 @@
+
+import requests
+import pandas as pd
+import json
+from datetime import datetime
+from read_benchmark_data import build_query_params
+
+def fetch_llm_data(payload: dict) -> list:
+    url = "https://hud.pytorch.org/api/clickhouse/oss_ci_benchmark_llms"
+    headers = {"Content-Type": "application/json"}
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    response.raise_for_status()
+    return response.json()
+
+def flatten_record(record):
+    flat = {
+        "timestamp": record.get("metadata_info", {}).get("timestamp"),
+        "workflow_id": record.get("workflow_id"),
+        "job_id": record.get("job_id"),
+        "model": record.get("model"),
+        "backend": record.get("backend"),
+        "mode": record.get("mode"),
+        "dtype": record.get("dtype"),
+        "device": record.get("device"),
+        "arch": record.get("arch"),
+        "granularity_bucket": record.get("granularity_bucket"),
+    }
+    if "extra" in record:
+        for k, v in record["extra"].items():
+            flat[f"extra_{k}"] = v
+    metric_name = record["metric"]
+    flat[metric_name] = record["actual"]
+    return flat
+
+def process_records(data: list) -> pd.DataFrame:
+    flattened = [flatten_record(entry) for entry in data]
+    df = pd.DataFrame(flattened)
+    id_cols = [
+        "timestamp", "workflow_id", "job_id", "model", "backend", "mode", "dtype",
+        "device", "arch", "granularity_bucket",
+        "extra_use_torch_compile", "extra_is_dynamic",
+        "extra_request_rate", "extra_tensor_parallel_size"
+    ]
+    df = df.groupby(id_cols, dropna=False).first().reset_index()
+    return df
+
+def main():
+    props = {
+        "archName": "All Platforms",
+        "deviceName": "Samsung Galaxy S22 5G (private) (Android 13)",
+        "modeName": "inference",
+        "modelName": "mv3",
+        "backendName": "qnn_q8",
+        "benchmarkName": "",  # fallback to repo default
+        "repoName": "pytorch/executorch",
+        "granularity": "hour",
+        "startTime": datetime(2025, 5, 23, 1, 1, 22),
+        "stopTime": datetime(2025, 6, 6, 1, 1, 22),
+        "branch": "main",
+        "commit": "098c58e1adc082ad98ffd6efb41151736fbc1a12"
+    }
+    dtypes = [""]
+
+    payload = build_query_params(props, dtypes)
+    print("Query payload:")
+    print(json.dumps(payload, indent=2))
+
+    data = fetch_llm_data(payload)
+    df = process_records(data)
+
+    df.to_csv("llm_benchmark_result.csv", index=False)
+    print(df.head())
+
+if __name__ == "__main__":
+    main()

.ci/scripts/benchmark_tooling/query_torch_analysis_data.py

@@ -0,0 +1,94 @@
+import datetime
+from pprint import pprint
+from typing import Any
+from dataclasses import dataclass, asdict
+import json
+import requests
+from urllib.parse import urlencode
+import argparse
+
+@dataclass
+class BenchmarkQueryGroupDataParams:
+    repo: str
+    benchmark_name: str
+    start_time: str
+    end_time: str
+    group_table_by_fields: list
+    group_row_by_fields: list
+
+BASE_URLS = {
+    "local": "http://localhost:3000",
+    "prod": "https://hud.pytorch.org",
+}
+
+def validate_iso8601_no_ms(value):
+    try:
+        # Only allow format without milliseconds
+        return datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%dT%H:%M:%S")
+    except ValueError:
+        raise argparse.ArgumentTypeError(
+            f"Invalid datetime format for '{value}'. Expected: YYYY-MM-DDTHH:MM:SS"
+        )
+
+def argparser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--env", choices=["local", "prod"], default="prod", help="Choose environment"
+    )
+    parser.add_argument(
+        "--startTime",
+        type=validate_iso8601_no_ms,
+        required=True,
+        help="Start time in ISO format (e.g. 2025-06-01T00:00:00)",
+    )
+    parser.add_argument(
+        "--endTime",
+        type=validate_iso8601_no_ms,
+        required=True,
+        help="End time in ISO format (e.g. 2025-06-06T00:00:00)",
+    )
+    return parser.parse_args()
+
+
+BASE_URLS = {
+    "local": "http://localhost:3000",
+    "prod": "https://hud.pytorch.org",
+}
+
+
+def fetch_execu_torch_data(startTime: str, endTime: str, env: str = 'prod'):
+    url = f"{BASE_URLS[env]}/api/benchmark/group_data/execuTorch"
+    # Convert back to string in the same format 2025-06-01T00:00:00
+    start_time_str = startTime
+    end_time_str = endTime
+
+    params_object = BenchmarkQueryGroupDataParams(
+        repo="pytorch/executorch",
+        benchmark_name="ExecuTorch",
+        start_time=start_time_str,
+        end_time=end_time_str,
+        group_table_by_fields=["device", "backend", "arch", "model"],
+        group_row_by_fields=["workflow_id", "job_id", "granularity_bucket"],
+    )
+
+    # Convert to JSON string
+    params = json.dumps(asdict(params_object))
+    response = requests.get(url, params=params)
+    if response.status_code == 200:
+        print("Successfully fetched benchmark data")
+        resp = response.json()
+        print(f"fetched {len(resp)} table views")
+        print(f"peeking first table view, peeking.... {resp[0]} ")
+    else:
+        print(f"Failed to fetch benchmark data ({response.status_code})")
+        print(response.text)
+
+
+
+
+def main():
+    args = argparser()
+    fetch_execu_torch_data(args.startTime, args.endTime, args.env)
+
+if __name__ == "__main__":
+    main()

.ci/scripts/benchmark_tooling/read_benchmark_data.py

@@ -0,0 +1,83 @@
+import pandas as pd
+
+DEFAULT_ARCH_NAME = "All Platforms"
+DEFAULT_DEVICE_NAME = "All Devices"
+DEFAULT_MODE_NAME = "All Modes"
+DEFAULT_MODEL_NAME = "All Models"
+DEFAULT_BACKEND_NAME = "All Backends"
+
+EXCLUDED_METRICS = [
+    "load_status", "mean_itl_ms", "mean_tpot_ms", "mean_ttft_ms",
+    "std_itl_ms", "std_tpot_ms", "std_ttft_ms",
+    "cold_compile_time(s)", "warm_compile_time(s)",
+    "speedup_pct", "generate_time(ms)"
+]
+
+REPO_TO_BENCHMARKS = {
+    "pytorch/executorch": ["ExecuTorch"],
+    # 其他 repo 可继续加
+}
+
+from datetime import datetime
+
+def format_time(dt: datetime) -> str:
+    return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
+
+from typing import Dict, Any
+
+def build_query_params(props: Dict[str, Any], dtypes: list) -> dict:
+    return {
+        "arch": "" if props["archName"] == DEFAULT_ARCH_NAME else props["archName"],
+        "device": "" if props["deviceName"] == DEFAULT_DEVICE_NAME else props["deviceName"],
+        "mode": "" if props["modeName"] == DEFAULT_MODE_NAME else props["modeName"],
+        "dtypes": dtypes,
+        "excludedMetrics": EXCLUDED_METRICS,
+        "benchmarks": [props["benchmarkName"]] if props.get("benchmarkName") else REPO_TO_BENCHMARKS.get(props["repoName"], []),
+        "granularity": props["granularity"],
+        "models": [] if props["modelName"] == DEFAULT_MODEL_NAME else [props["modelName"]],
+        "backends": [] if props["backendName"] == DEFAULT_BACKEND_NAME else [props["backendName"]],
+        "repo": props["repoName"],
+        "startTime": format_time(props["startTime"]),
+        "stopTime": format_time(props["stopTime"]),
+    }
+
+def flatten_record(record):
+    flat = {
+        "timestamp": record.get("metadata_info", {}).get("timestamp"),
+        "workflow_id": record.get("workflow_id"),
+        "job_id": record.get("job_id"),
+        "model": record.get("model"),
+        "backend": record.get("backend"),
+        "mode": record.get("mode"),
+        "dtype": record.get("dtype"),
+        "device": record.get("device"),
+        "arch": record.get("arch"),
+        "granularity_bucket": record.get("granularity_bucket"),
+    }
+
+    # Flatten extra
+    if "extra" in record:
+        for k, v in record["extra"].items():
+            flat[f"extra_{k}"] = v
+
+    # Add metric-specific value
+    metric_name = record["metric"]
+    flat[metric_name] = record["actual"]
+
+    return flat
+
+def process_records(data: list) -> pd.DataFrame:
+    flattened = [flatten_record(entry) for entry in data]
+    df = pd.DataFrame(flattened)
+
+    # Group by workflow_id, job_id, timestamp
+    id_cols = [
+        "timestamp", "workflow_id", "job_id", "model", "backend", "mode", "dtype",
+        "device", "arch", "granularity_bucket",
+        "extra_use_torch_compile", "extra_is_dynamic",
+        "extra_request_rate", "extra_tensor_parallel_size"
+    ]
+
+    df = df.groupby(id_cols, dropna=False).first().reset_index()
+
+    return df

.ci/scripts/benchmark_tooling/requirements.txt

@@ -0,0 +1 @@
+requests>=2.32.3