Skip to content

Commit ba3fd8f

Browse files
committed
Add A/B testing command line arguments
Core A/B Testing Features: - Add --side-a/-a and --side-b/-b params - Validation for single operator requirement - Error messages for wrong usage Example usage: python run.py --op vector_add --side-a "--precision fp16" --side-b "--precision fp32"
1 parent 5c093a3 commit ba3fd8f

File tree

1 file changed

+33
-0
lines changed

1 file changed

+33
-0
lines changed

tritonbench/utils/parser.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,24 @@ def get_parser(args=None):
246246
default=None,
247247
help="Name of group for benchmarking.",
248248
)
249+
250+
# A/B Testing parameters
251+
parser.add_argument(
252+
"--side-a",
253+
"-a",
254+
type=str,
255+
default=None,
256+
help="Configuration A for A/B testing. Specify operator-specific arguments as a string. "
257+
"Example: '--side-a \"--max-autotune --dynamic\"'",
258+
)
259+
parser.add_argument(
260+
"--side-b",
261+
"-b",
262+
type=str,
263+
default=None,
264+
help="Configuration B for A/B testing. Specify operator-specific arguments as a string. "
265+
"Example: '--side-b \"--dynamic\"'",
266+
)
249267

250268
if is_fbcode():
251269
parser.add_argument("--log-scuba", action="store_true", help="Log to scuba.")
@@ -268,4 +286,19 @@ def get_parser(args=None):
268286
print(
269287
"Neither operator nor operator collection is specified. Running all operators in the default collection."
270288
)
289+
290+
# A/B Testing validation
291+
if (args.side_a is not None) != (args.side_b is not None):
292+
parser.error("A/B testing requires both --side-a and --side-b arguments to be specified together")
293+
294+
if args.side_a is not None and args.side_b is not None:
295+
# A/B mode is enabled
296+
if not args.op:
297+
parser.error("A/B testing requires a specific operator (--op) to be specified")
298+
if args.op_collection != "default":
299+
parser.error("A/B testing is only supported with single operators, not operator collections")
300+
if "," in args.op:
301+
parser.error("A/B testing is only supported with a single operator, not multiple operators")
302+
if args.isolate:
303+
parser.error("A/B testing is not compatible with --isolate mode")
271304
return parser

0 commit comments

Comments
 (0)