|
| 1 | +#!/usr/bin/env python |
| 2 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | +# All rights reserved. |
| 4 | +# |
| 5 | +# This source code is licensed under the BSD-style license found in the |
| 6 | +# LICENSE file in the root directory of this source tree. |
| 7 | + |
| 8 | +import json |
| 9 | +import logging |
| 10 | +import os |
| 11 | +import re |
| 12 | +from typing import Any, Dict |
| 13 | + |
| 14 | +from examples.models import MODEL_NAME_TO_MODEL |
| 15 | + |
| 16 | + |
| 17 | +# Device pools for AWS Device Farm |
| 18 | +DEVICE_POOLS = { |
| 19 | + "apple_iphone_15": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3b5acd2e-92e2-4778-b651-7726bafe129d", |
| 20 | + "apple_iphone_15+ios_18": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/12c8b15c-8d03-4e07-950d-0a627e7595b4", |
| 21 | + "samsung_galaxy_s22": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa", |
| 22 | + "samsung_galaxy_s24": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98f8788c-2e25-4a3c-8bb2-0d1e8897c0db", |
| 23 | + "google_pixel_8_pro": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d65096ab-900b-4521-be8b-a3619b69236a", |
| 24 | +} |
| 25 | + |
| 26 | +# Predefined benchmark configurations |
| 27 | +BENCHMARK_CONFIGS = { |
| 28 | + "xplat": [ |
| 29 | + "xnnpack_q8", |
| 30 | + "hf_xnnpack_fp32", |
| 31 | + "llama3_fb16", |
| 32 | + "llama3_spinquant", |
| 33 | + "llama3_qlora", |
| 34 | + ], |
| 35 | + "android": [ |
| 36 | + "qnn_q8", |
| 37 | + # TODO: Add support for llama3 htp |
| 38 | + # "llama3_qnn_htp", |
| 39 | + ], |
| 40 | + "ios": [ |
| 41 | + "coreml_fp16", |
| 42 | + "mps", |
| 43 | + "llama3_coreml_ane", |
| 44 | + ], |
| 45 | +} |
| 46 | + |
| 47 | + |
| 48 | +def parse_args() -> Any: |
| 49 | + """ |
| 50 | + Parse command-line arguments. |
| 51 | +
|
| 52 | + Returns: |
| 53 | + argparse.Namespace: Parsed command-line arguments. |
| 54 | +
|
| 55 | + Example: |
| 56 | + parse_args() -> Namespace(models=['mv3', 'meta-llama/Llama-3.2-1B-Instruct-QLORA_INT4_EO8'], |
| 57 | + os='android', |
| 58 | + devices=['samsung_galaxy_s22']) |
| 59 | + """ |
| 60 | + from argparse import ArgumentParser |
| 61 | + |
| 62 | + def comma_separated(value: str): |
| 63 | + """ |
| 64 | + Parse a comma-separated string into a list. |
| 65 | + """ |
| 66 | + return value.split(",") |
| 67 | + |
| 68 | + parser = ArgumentParser("Gather all benchmark configs.") |
| 69 | + parser.add_argument( |
| 70 | + "--os", |
| 71 | + type=str, |
| 72 | + choices=["android", "ios"], |
| 73 | + help="The target OS.", |
| 74 | + ) |
| 75 | + parser.add_argument( |
| 76 | + "--models", |
| 77 | + type=comma_separated, # Use the custom parser for comma-separated values |
| 78 | + help=f"Comma-separated model IDs or names. Valid values include {MODEL_NAME_TO_MODEL}.", |
| 79 | + ) |
| 80 | + parser.add_argument( |
| 81 | + "--devices", |
| 82 | + type=comma_separated, # Use the custom parser for comma-separated values |
| 83 | + help=f"Comma-separated device names. Available devices: {list(DEVICE_POOLS.keys())}", |
| 84 | + ) |
| 85 | + |
| 86 | + return parser.parse_args() |
| 87 | + |
| 88 | + |
| 89 | +def set_output(name: str, val: Any) -> None: |
| 90 | + """ |
| 91 | + Set the output value to be used by other GitHub jobs. |
| 92 | +
|
| 93 | + Args: |
| 94 | + name (str): The name of the output variable. |
| 95 | + val (Any): The value to set for the output variable. |
| 96 | +
|
| 97 | + Example: |
| 98 | + set_output("benchmark_configs", {"include": [...]}) |
| 99 | + """ |
| 100 | + |
| 101 | + if os.getenv("GITHUB_OUTPUT"): |
| 102 | + print(f"Setting {val} to GitHub output") |
| 103 | + with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env: |
| 104 | + print(f"{name}={val}", file=env) |
| 105 | + else: |
| 106 | + print(f"::set-output name={name}::{val}") |
| 107 | + |
| 108 | + |
| 109 | +def is_valid_huggingface_model_id(model_name: str) -> bool: |
| 110 | + """ |
| 111 | + Validate if the model name matches the pattern for HuggingFace model IDs. |
| 112 | +
|
| 113 | + Args: |
| 114 | + model_name (str): The model name to validate. |
| 115 | +
|
| 116 | + Returns: |
| 117 | + bool: True if the model name matches the valid pattern, False otherwise. |
| 118 | +
|
| 119 | + Example: |
| 120 | + is_valid_huggingface_model_id('meta-llama/Llama-3.2-1B') -> True |
| 121 | + """ |
| 122 | + pattern = r"^[a-zA-Z0-9-_]+/[a-zA-Z0-9-_.]+$" |
| 123 | + return bool(re.match(pattern, model_name)) |
| 124 | + |
| 125 | + |
| 126 | +def get_benchmark_configs() -> Dict[str, Dict]: |
| 127 | + """ |
| 128 | + Gather benchmark configurations for a given set of models on the target operating system and devices. |
| 129 | +
|
| 130 | + Args: |
| 131 | + None |
| 132 | +
|
| 133 | + Returns: |
| 134 | + Dict[str, Dict]: A dictionary containing the benchmark configurations. |
| 135 | +
|
| 136 | + Example: |
| 137 | + get_benchmark_configs() -> { |
| 138 | + "include": [ |
| 139 | + { |
| 140 | + "model": "meta-llama/Llama-3.2-1B", |
| 141 | + "config": "llama3_qlora", |
| 142 | + "device_name": "apple_iphone_15", |
| 143 | + "device_arn": "arn:aws:..." |
| 144 | + }, |
| 145 | + { |
| 146 | + "model": "mv3", |
| 147 | + "config": "xnnpack_q8", |
| 148 | + "device_name": "samsung_galaxy_s22", |
| 149 | + "device_arn": "arn:aws:..." |
| 150 | + }, |
| 151 | + ... |
| 152 | + ] |
| 153 | + } |
| 154 | + """ |
| 155 | + args = parse_args() |
| 156 | + target_os = args.os |
| 157 | + devices = args.devices |
| 158 | + models = args.models |
| 159 | + |
| 160 | + benchmark_configs = {"include": []} |
| 161 | + |
| 162 | + for model_name in models: |
| 163 | + configs = [] |
| 164 | + if is_valid_huggingface_model_id(model_name): |
| 165 | + if model_name.startswith("meta-llama/"): |
| 166 | + # LLaMA models |
| 167 | + repo_name = model_name.split("meta-llama/")[1] |
| 168 | + if "qlora" in repo_name.lower(): |
| 169 | + configs.append("llama3_qlora") |
| 170 | + elif "spinquant" in repo_name.lower(): |
| 171 | + configs.append("llama3_spinquant") |
| 172 | + else: |
| 173 | + configs.append("llama3_fb16") |
| 174 | + configs.extend( |
| 175 | + [ |
| 176 | + config |
| 177 | + for config in BENCHMARK_CONFIGS.get(target_os, []) |
| 178 | + if config.startswith("llama") |
| 179 | + ] |
| 180 | + ) |
| 181 | + else: |
| 182 | + # Non-LLaMA models |
| 183 | + configs.append("hf_xnnpack_fp32") |
| 184 | + elif model_name in MODEL_NAME_TO_MODEL: |
| 185 | + # ExecuTorch in-tree non-GenAI models |
| 186 | + configs.append("xnnpack_q8") |
| 187 | + configs.extend( |
| 188 | + [ |
| 189 | + config |
| 190 | + for config in BENCHMARK_CONFIGS.get(target_os, []) |
| 191 | + if not config.startswith("llama") |
| 192 | + ] |
| 193 | + ) |
| 194 | + else: |
| 195 | + # Skip unknown models with a warning |
| 196 | + logging.warning(f"Unknown or invalid model name '{model_name}'. Skipping.") |
| 197 | + continue |
| 198 | + |
| 199 | + # Add configurations for each valid device |
| 200 | + for device in devices: |
| 201 | + for config in configs: |
| 202 | + if config == "llama3_coreml_ane" and not device.endswith("+ios_18"): |
| 203 | + device = f"{device}+ios_18" |
| 204 | + logging.info( |
| 205 | + f"Benchmark config '{config}' only works on iOS 18+, auto-upgraded device pool to '{device}'" |
| 206 | + ) |
| 207 | + |
| 208 | + if device not in DEVICE_POOLS: |
| 209 | + logging.warning(f"Unsupported device '{device}'. Skipping.") |
| 210 | + continue |
| 211 | + |
| 212 | + record = { |
| 213 | + "model": model_name, |
| 214 | + "config": config, |
| 215 | + "device_name": device, |
| 216 | + "device_arn": DEVICE_POOLS[device], |
| 217 | + } |
| 218 | + benchmark_configs["include"].append(record) |
| 219 | + |
| 220 | + set_output("benchmark_configs", json.dumps(benchmark_configs)) |
| 221 | + |
| 222 | + |
| 223 | +if __name__ == "__main__": |
| 224 | + get_benchmark_configs() |
0 commit comments