final

Signed-off-by: Yang Wang <[email protected]>

Author: yangw-dev

.ci/scripts/benchmark_tooling/README.md

@@ -30,21 +30,24 @@ python3 .ci/scripts/benchmark_tooling/get_benchmark_analysis_data.py \
 - `--startTime`: Start time in ISO format (e.g., "2025-06-11T00:00:00") (required)
 - `--endTime`: End time in ISO format (e.g., "2025-06-17T18:00:00") (required)
 - `--env`: Choose environment ("local" or "prod", default: "prod")
-- `--not-silent`: Show processing logs (default: only show results & minimum logging)
+- `--no-silent`: Show processing logs (default: only show results & minimum logging)
+- `print-all-table-info`: show all cleaned table infos, this helps user to pick the correct format of filters
 
 ##### Output Options:
 - `--outputType`: Choose output format (default: "print")
   - `print`: Display results in console
   - `json`: Generate JSON file
   - `df`: Display results in DataFrame format
-  - `excel`: Generate Excel files with multiple sheets
-  - `csv`: Generate CSV files in separate folders
+  - `excel`: Generate Excel files with multiple sheets, the field in first row and first column contains the json string of the raw metadata
+  - `csv`: Generate CSV files in separate folders, the field in first row and first column contains the json string of the raw metadata
 - `--outputDir`: Directory to save output files (default: current directory)
 
 ##### Filtering Options:
+Notice, the filter needs full name matchings with correct format, to see all the options of the filter choices, please run the script with `--print-all-table-info`, and pay attention to section `Full list of table info from HUD API` with the field 'info', which contains normalized data we use to filter records from the original metadata 'groupInfo'.
+
 - `--devices`: Filter by specific device names (e.g., "samsung-galaxy-s22-5g", "samsung-galaxy-s22plus-5g")
-- `--backends`: Filter by specific backend names
-- `--models`: Filter by specific model names
+- `--backends`: Filter by specific backend names (e.g.,  "qnn-q8" , ""llama3-spinquan)
+- `--models`: Filter by specific model names (e.g "mv3" "meta-llama-llama-3.2-1b-instruct-qlora-int4-eo8")
 
 #### Working with Output Files
 

.ci/scripts/benchmark_tooling/get_benchmark_analysis_data.py

@@ -456,7 +456,7 @@ def _get_base_url(self) -> str:
         }
         return base_urls[self.env]
 
-    def print_all_groups_info(self) -> None:
+    def print_all_table_info(self) -> None:
         """
         Print all benchmark table group info found in the data.
         Separates results by category and displays counts.
@@ -670,6 +670,13 @@ def argparsers():
         default=True,
         help="Allow output (disable silent mode)",
     )
+
+    parser.add_argument(
+        "--print-all-table-info",
+        action="store_true",
+        help="Print all table info for debugging",
+    )
+
     # Options for generate_data
     parser.add_argument(
         "--outputType",
@@ -689,9 +696,11 @@ def argparsers():
     parser.add_argument(
         "--devices",
         nargs="+",
-        help="Filter results by device names (e.g. --devices samsung-galaxy-s22-5g)(OR logic)",
+        help="Filter results by one or more device names (e.g. --devices samsung-galaxy-s22-5g)(OR logic)",
+    )
+    parser.add_argument(
+        "--models", nargs="+", help="Filter by one or more models (OR logic)"
     )
-    parser.add_argument("--models", nargs="+", help="Filter by models (OR logic)")
     return parser.parse_args()
 
 
@@ -705,6 +714,6 @@ def argparsers():
             models=args.models, backends=args.backends, devices=args.devices
         ),
     )
-    if not args.silent:
-        fetcher.print_all_groups_info()
+    if args.print_all_table_info:
+        fetcher.print_all_table_info()
     fetcher.output_data(args.outputType, args.outputDir)