Skip to content
This repository was archived by the owner on Apr 23, 2026. It is now read-only.

Commit b281743

Browse files
google-labs-jules[bot]insign
google-labs-jules[bot]
and
insign
committed
Update benchmark scores for various models
Sources verified: - LiveBench leaderboard (livebench_data.txt) - SWE-Bench Verified scores (swebench_data.txt) - HLE leaderboard (hle_data.txt) - FrontierMath leaderboard (epoch.ai/frontiermath) - BFCL leaderboard (gorilla.cs.berkeley.edu/leaderboard.html) - OSWorld leaderboard (os-world.github.io) - MMMU-Pro leaderboard (mmmu-benchmark.github.io) Changes include updating livebench, swe_bench, humanity_last_exam, frontiermath, bfcl, osworld, and mmmu_pro values for recent models such as Claude Opus/Sonnet 4.5/4.6, GPT-5 series, Gemini 3 series, DeepSeek V3.2, Grok 4.1, Kimi K2.5, and o3. Note: Single score reported for SWE-Bench families were assigned to their thinking variants per UPDATE.md guidelines. Base models were set to null for automatic inferior_of imputation. Co-authored-by: insign <1113045+insign@users.noreply.github.com>
1 parent 920f56b commit b281743

1 file changed

Lines changed: 35 additions & 33 deletions

File tree

‎data/showdown.json‎

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"meta": {
33
"version": "2026.04.16",
4-
"last_update": "2026-04-18T19:00:00Z",
4+
"last_update": "2026-04-19T15:45:05Z",
55
"schema_version": "1.0"
66
},
77
"models": [
@@ -152,7 +152,7 @@
152152
"livebench": null,
153153
"math_500": null,
154154
"aime": null,
155-
"frontiermath": null,
155+
"frontiermath": 43.8,
156156
"arc_agi_2": null,
157157
"bfcl": null,
158158
"tau_bench": null,
@@ -257,11 +257,11 @@
257257
"aime": null,
258258
"arc_agi_2": 64.6,
259259
"bfcl": null,
260-
"frontiermath": null,
260+
"frontiermath": 38.3,
261261
"gpqa_diamond": null,
262262
"humanity_last_exam": 40,
263263
"live_code_bench": null,
264-
"livebench": null,
264+
"livebench": 61.81,
265265
"lmarena_coding_elo": 1547,
266266
"lmarena_creative_elo": 1468,
267267
"lmarena_en_elo": 1497,
@@ -377,7 +377,7 @@
377377
"benchmark_scores": {
378378
"aime": 87,
379379
"arc_agi_2": 37.6,
380-
"bfcl": 82.5,
380+
"bfcl": 77.47,
381381
"frontiermath": 20.7,
382382
"gpqa_diamond": 87,
383383
"humanity_last_exam": 25.2,
@@ -397,9 +397,9 @@
397397
"mmmlu": 90.8,
398398
"simpleqa": 30,
399399
"mmmu": 80.7,
400-
"mmmu_pro": 80.7,
400+
"mmmu_pro": 73.9,
401401
"osworld": 61.4,
402-
"swe_bench": 80.9,
402+
"swe_bench": null,
403403
"tau_bench": 70.2,
404404
"terminal_bench": 59.3,
405405
"webdev_arena_elo": 1479,
@@ -465,7 +465,7 @@
465465
"mmmu": 83,
466466
"mmmu_pro": 80.7,
467467
"osworld": 66.3,
468-
"swe_bench": 80.9,
468+
"swe_bench": 76.8,
469469
"swe_bench_pro": 45.89,
470470
"tau_bench": 70.2,
471471
"terminal_bench": 59.3,
@@ -527,8 +527,8 @@
527527
"mmmlu": 89.3,
528528
"simpleqa": null,
529529
"mmmu": null,
530-
"mmmu_pro": null,
531-
"osworld": null,
530+
"mmmu_pro": 74.5,
531+
"osworld": 72.1,
532532
"swe_bench": null,
533533
"tau_bench": null,
534534
"terminal_bench": null,
@@ -572,7 +572,7 @@
572572
"aime": 91.2,
573573
"arc_agi_2": 60.4,
574574
"bfcl": null,
575-
"frontiermath": null,
575+
"frontiermath": 32.4,
576576
"gpqa_diamond": 89.9,
577577
"humanity_last_exam": 33.2,
578578
"live_code_bench": null,
@@ -591,7 +591,7 @@
591591
"mmmlu": 89.3,
592592
"simpleqa": null,
593593
"mmmu": null,
594-
"mmmu_pro": 74.5,
594+
"mmmu_pro": 75.6,
595595
"osworld": 72.5,
596596
"swe_bench": 79.6,
597597
"tau_bench": null,
@@ -628,12 +628,12 @@
628628
"benchmark_scores": {
629629
"aime": 87,
630630
"arc_agi_2": 13.6,
631-
"bfcl": 80.8,
631+
"bfcl": 73.24,
632632
"frontiermath": 9.3,
633633
"gpqa_diamond": 83.4,
634634
"humanity_last_exam": 13.7,
635635
"live_code_bench": 74.5,
636-
"livebench": 70.31,
636+
"livebench": 53.69,
637637
"lmarena_coding_elo": 1510,
638638
"lmarena_creative_elo": 1445,
639639
"lmarena_en_elo": 1451,
@@ -648,9 +648,9 @@
648648
"mmmlu": 89.1,
649649
"simpleqa": 29.3,
650650
"mmmu": 66.2,
651-
"mmmu_pro": 68,
652-
"osworld": 61.4,
653-
"swe_bench": 77.2,
651+
"mmmu_pro": 68.9,
652+
"osworld": 53.6,
653+
"swe_bench": null,
654654
"tau_bench": 58.5,
655655
"terminal_bench": 50,
656656
"webdev_arena_elo": 1387,
@@ -716,7 +716,7 @@
716716
"mmmu": 70.2,
717717
"mmmu_pro": 68,
718718
"osworld": 55.8,
719-
"swe_bench": 77.2,
719+
"swe_bench": 71.4,
720720
"swe_bench_pro": 43.6,
721721
"tau_bench": 62.5,
722722
"terminal_bench": 63.5,
@@ -952,7 +952,7 @@
952952
"bfcl": 84.5,
953953
"frontiermath": 31,
954954
"gpqa_diamond": 88.1,
955-
"humanity_last_exam": 26.5,
955+
"humanity_last_exam": 23.68,
956956
"live_code_bench": 78,
957957
"livebench": 72.04,
958958
"lmarena_coding_elo": 1491,
@@ -969,7 +969,7 @@
969969
"mmmlu": 91,
970970
"simpleqa": 34.9,
971971
"mmmu": 85.4,
972-
"mmmu_pro": 76,
972+
"mmmu_pro": 79.0,
973973
"osworld": null,
974974
"swe_bench": 76.3,
975975
"tau_bench": null,
@@ -1004,7 +1004,7 @@
10041004
},
10051005
"editor_notes": "OpenAI's latest standard model. Blazing fast inference at 187 TPS. Excellent balance of speed and intelligence.",
10061006
"benchmark_scores": {
1007-
"swe_bench": 80,
1007+
"swe_bench": null,
10081008
"arc_agi_2": 43.3,
10091009
"lmarena_en_elo": 1439,
10101010
"lmarena_coding_elo": 1497,
@@ -1014,7 +1014,9 @@
10141014
"lmarena_creative_elo": 1396,
10151015
"lmarena_if_elo": 1429,
10161016
"lmarena_vision_elo": 1229,
1017-
"webdev_arena_elo": 1403
1017+
"webdev_arena_elo": 1403,
1018+
"humanity_last_exam": 27.8,
1019+
"frontiermath": 36.9
10181020
}
10191021
},
10201022
{
@@ -1059,7 +1061,7 @@
10591061
"mmmlu": 89.5,
10601062
"simpleqa": 38.2,
10611063
"mmmu": 76.5,
1062-
"mmmu_pro": 86.5,
1064+
"mmmu_pro": 79.5,
10631065
"osworld": 58.5,
10641066
"swe_bench": 72.8,
10651067
"swe_bench_pro": 55.6,
@@ -1348,10 +1350,10 @@
13481350
"benchmark_scores": {
13491351
"aime": 95,
13501352
"arc_agi_2": 31.1,
1351-
"bfcl": 82.8,
1353+
"bfcl": 72.51,
13521354
"frontiermath": 37.6,
13531355
"gpqa_diamond": 91.9,
1354-
"humanity_last_exam": 37.5,
1356+
"humanity_last_exam": 37.52,
13551357
"live_code_bench": 78.5,
13561358
"livebench": 73.46,
13571359
"lmarena_coding_elo": 1519,
@@ -1792,7 +1794,7 @@
17921794
"mmmu": 62.5,
17931795
"mmmu_pro": 83.7,
17941796
"osworld": 38.5,
1795-
"swe_bench": 66,
1797+
"swe_bench": null,
17961798
"tau_bench": 55.2,
17971799
"terminal_bench": 31.3,
17981800
"webdev_arena_elo": 1285,
@@ -1827,7 +1829,7 @@
18271829
"benchmark_scores": {
18281830
"aime": 88.4,
18291831
"arc_agi_2": 8.5,
1830-
"bfcl": 80.5,
1832+
"bfcl": 56.73,
18311833
"frontiermath": 22.1,
18321834
"gpqa_diamond": 80.1,
18331835
"humanity_last_exam": 15.9,
@@ -2053,7 +2055,7 @@
20532055
"benchmark_scores": {
20542056
"aime": 84,
20552057
"arc_agi_2": 15.9,
2056-
"bfcl": 83.5,
2058+
"bfcl": 58.29,
20572059
"frontiermath": 19.7,
20582060
"gpqa_diamond": 87,
20592061
"humanity_last_exam": 44.4,
@@ -2110,7 +2112,7 @@
21102112
"benchmark_scores": {
21112113
"aime": 84,
21122114
"arc_agi_2": 26,
2113-
"bfcl": 85.5,
2115+
"bfcl": 69.57,
21142116
"frontiermath": 19.7,
21152117
"gpqa_diamond": 87,
21162118
"humanity_last_exam": 50.7,
@@ -2538,7 +2540,7 @@
25382540
"benchmark_scores": {
25392541
"aime": 83.9,
25402542
"arc_agi_2": 6.5,
2541-
"bfcl": 78.5,
2543+
"bfcl": 63.05,
25422544
"frontiermath": 18.7,
25432545
"gpqa_diamond": 81.8,
25442546
"humanity_last_exam": 24,
@@ -2897,7 +2899,7 @@
28972899
"benchmark_scores": {
28982900
"aime": 83.1,
28992901
"arc_agi_2": 2.5,
2900-
"bfcl": 90,
2902+
"bfcl": 59.06,
29012903
"frontiermath": 21.4,
29022904
"gpqa_diamond": 84.2,
29032905
"humanity_last_exam": 23.9,
@@ -2958,11 +2960,11 @@
29582960
"gpqa_diamond": null,
29592961
"arc_agi_2": null,
29602962
"livebench": null,
2961-
"humanity_last_exam": null,
2963+
"humanity_last_exam": 24.37,
29622964
"lmarena_hard_elo": 1470,
29632965
"bfcl": null,
29642966
"tau_bench": null,
2965-
"osworld": null,
2967+
"osworld": 63.3,
29662968
"webdev_arena_elo": null,
29672969
"lmarena_creative_elo": null,
29682970
"lmarena_if_elo": null,

0 commit comments

Comments
 (0)