Skip to content

Commit a74e3cb

Browse files
committed
fix submission for VLM after breaking changes in PR: mlcommons#2397
Signed-off-by: John Calderon <[email protected]>
1 parent a4e012a commit a74e3cb

File tree

1 file changed

+3
-1
lines changed

1 file changed

+3
-1
lines changed

tools/submission/submission_checker/constants.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@
185185
# TODO: Placeholder for now
186186
"gpt-oss-120b": ("exact_match", 83.13 * 0.99),
187187
# TODO: Placeholder for now
188-
"qwen3-vl-235b-a22b": ("F1", 0.7903 * 0.99),
188+
"qwen3-vl-235b-a22b": ("F1_HIERARCHICAL", 0.7903 * 0.99),
189189
"dlrm-v3": ("AUC", 78.663 * 0.99), # TODO: Placeholder for now
190190
"yolo-95": ("mAP", 53.4 * 0.95),
191191
"yolo-99": ("mAP", 53.4 * 0.99),
@@ -1153,6 +1153,7 @@
11531153
"pointpainting": 6636,
11541154
"yolo-99": 1525,
11551155
"yolo-95": 1525,
1156+
"qwen3-vl-235b-a22b": 48289
11561157
}
11571158

11581159
SCENARIO_MAPPING = {
@@ -1387,6 +1388,7 @@
13871388
"mbxp_accuracy": r".*'mbxp':\s([\d.]+).*",
13881389
"exact_match": r".*'exact_match':\s([\d.]+).*",
13891390
"vbench_score": r".*'vbench_score':\s([\d.]+).*",
1391+
"F1_HIERARCHICAL": r'\{.*"f1":\s*([\d\.]+).*\}',
13901392
}
13911393

13921394
SYSTEM_DESC_REQUIRED_FIELDS = [

0 commit comments

Comments
 (0)