setup link

Signed-off-by: Yang Wang <[email protected]>

Author: yangw-dev

.ci/scripts/benchmark_tooling/README.md

@@ -52,7 +52,7 @@ python3 .ci/scripts/benchmark_tooling/get_benchmark_analysis_data.py \
   --endTime "2025-06-17T18:00:00" \
   --outputType "excel"
 
-python3 analyze_benchmark_stability.py \
+python3 .ci/scripts/benchmark_tooling/analyze_benchmark_stability.py \
 --primary-file private.xlsx \
 --reference-file public.xlsx
 ```

.ci/scripts/benchmark_tooling/analyze_benchmark_stability.py

@@ -1,6 +1,5 @@
 import argparse
 import os
-import re
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -16,35 +15,37 @@ def print_section_header(title):
     print("=" * 100 + "\n")
 
 
-def normalize_tab_name(name):
+def normalize_name(name):
     """Normalize tab name for better matching"""
     # Convert to lowercase and remove spaces
-    return name.lower().replace(" ", "")
+    return name.lower().replace(" ", "").replace("(private)", "")
 
 
 def parse_model_device_config(config):
     """Extract model and device from config"""
     model = config.get("model", "")
+    backend = config.get("backend", "")
+    full_model = f"{model}({backend})" if backend else model
     base_device = config.get("device", "")
     os_version = config.get("arch", "")
     full_device = f"{base_device}({os_version})" if os_version else base_device
     if not base_device:
-        return model, "unkown", "unknown", ""
-    return model, full_device, base_device, os_version
+        return full_model, "unkown", "unknown", ""
+    return full_model, full_device, base_device, os_version
 
 
 def is_matching_dataset(primary_sheet, reference_sheet):
     """
     Check if two datasets match for comparison based on model and device
     Allows different OS versions for the same device
     """
-    primary_model = primary_sheet.get("model", "")
-    primary_device = primary_sheet.get("base_device", "")
-    primary_os = primary_sheet.get("os_version", "")
+    primary_model = normalize_name(primary_sheet.get("model", ""))
+    primary_device = normalize_name(primary_sheet.get("base_device", ""))
+    # primary_os = normalize_name(primary_sheet.get("os_version", ""))
 
-    reference_model = reference_sheet.get("model", "")
-    reference_device = reference_sheet.get("base_device", "")
-    reference_os = reference_sheet.get("os_version", "")
+    reference_model = normalize_name(reference_sheet.get("model", ""))
+    reference_device = normalize_name(reference_sheet.get("base_device", ""))
+    # reference_os = normalize_name(reference_sheet.get("os_version", ""))
 
     if not primary_model:
         print("Warning: Primary sheet {} has no model info, for {primary_model} ")
@@ -82,9 +83,9 @@ def analyze_latency_stability(  # noqa: C901
 
     for document in documents:
         sheetName = document.get("sheetName", None)
-        print(f"Loading dataset: {sheetName}")
         df = document.get("df", None)
         config = document.get("groupInfo", None)
+        print(f"Loading dataset: {sheetName} with config: {config} ")
 
         if df is None or df.empty:
             print(f"Skipping sheet {sheetName} because it has no df data")
@@ -139,14 +140,13 @@ def analyze_latency_stability(  # noqa: C901
     reference_datasets = {}
     if reference_file:
         print_section_header("LOADING REFERENCE DATASETS (Public)")
-        documents = read_excel_with_json_header(primary_file)
+        documents = read_excel_with_json_header(reference_file)
 
         for document in documents:
             sheetName = document.get("sheetName", None)
-            print(f"Loading dataset: {sheetName}")
             df = document.get("df", None)
             config = document.get("groupInfo", None)
-
+            print(f"Loading dataset: {sheetName} with config:{config}")
             if df is None or df.empty:
                 print(f"Skipping sheet {sheetName} because it has no df data")
                 continue
@@ -263,7 +263,9 @@ def analyze_latency_stability(  # noqa: C901
                     break
 
             if not found_match:
-                print(f"Warning: No matching reference dataset for {primary_sheet}")
+                print(
+                    f"Warning: No matching reference dataset for {primary_sheet} with config:  {primary_info['model']}{primary_info['full_device']} "
+                )
 
         if not matches_found:
             print("No matching datasets found between primary and reference files.")

.ci/scripts/benchmark_tooling/get_benchmark_analysis_data.py

@@ -302,6 +302,7 @@ def output_data(
         logging.info(
             f"Generating output with type {output_type}: {[self.matching_groups.keys()]}"
         )
+
         o_type = self._to_output_type(output_type)
         if o_type == OutputType.PRINT:
             logging.info("\n ========= Generate print output ========= \n")
@@ -576,7 +577,10 @@ def _process_private_public_data(self, filters: Optional[BenchmarkFilters]):
         public_list = self._filter_public_result(private_list, all_public)
 
         logging.info(
-            f"Found {len(private_list)} private tables, Found assoicated {len(public_list)} public tables"
+            f"Found {len(private_list)} private tables, {[item['table_name'] for item in private_list]}"
+        )
+        logging.info(
+            f"Found assoicated {len(public_list)} public tables, {json.dumps([item['table_name'] for item in public_list],indent=2)}"
         )
 
         self.matching_groups["private"] = MatchingGroupResult(

backends/xnnpack/third-party/generate-cpuinfo-wrappers.py

@@ -66,7 +66,8 @@
     ],
     "(defined(__arm__) || defined(__aarch64__)) && defined(TARGET_OS_MAC) && TARGET_OS_MAC": [
         "arm/mach/init.c",
-    ],}
+    ],
+}
 
 
 if __name__ == "__main__":
@@ -77,7 +78,10 @@
                 print(filepath)
                 os.makedirs(os.path.dirname(filepath))
             with open(filepath, "w") as wrapper:
-                print("/* Auto-generated by generate-wrappers.py script. Do not modify */", file=wrapper)
+                print(
+                    "/* Auto-generated by generate-wrappers.py script. Do not modify */",
+                    file=wrapper,
+                )
                 print(file=wrapper)
                 print("#ifdef __APPLE__", file=wrapper)
                 print("\t#include <TargetConditionals.h>", file=wrapper)

extension/llm/custom_ops/spinquant/third-party/FFHT/example.py

@@ -15,6 +15,6 @@
 t2 = timeit.default_timer()
 
 if sys.version_info[0] == 2:
-    print (t2 - t1 + 0.0) / (reps + 0.0)
+    print(t2 - t1 + 0.0) / (reps + 0.0)
 if sys.version_info[0] == 3:
-    print('{}'.format((t2 - t1 + 0.0) / (reps + 0.0)))
+    print("{}".format((t2 - t1 + 0.0) / (reps + 0.0)))