llm-benchmark/src/benchmark-config.json at 6bfef62177cd327617c0a80347562939c86ffeda · tinybirdco/llm-benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
{
  "providers": {
    "x-ai": {
      "models": [
        "grok-3-beta",
        "grok-3-mini-beta",
        "grok-4",
        "grok-code-fast-1",
        "grok-4-fast"
      ]
    },
    "anthropic": {
      "models": [
        "claude-3.5-sonnet",
        "claude-3.7-sonnet",
        "claude-opus-4",
        "claude-sonnet-4",
        "claude-3.5-haiku",
        "claude-opus-4.1"
      ]
    },
    "deepseek": {
      "models": [
        "deepseek-chat-v3-0324",
        "deepseek-chat-v3-0324:free",
        "deepseek-chat-v3.1",
        "deepseek-v3.2-exp"
      ]
    },
    "google": {
      "models": [
        "gemini-2.0-flash-001",
        "gemini-2.5-flash",
        "gemini-2.5-pro"
      ]
    },
    "meta-llama": {
      "models": [
        "llama-4-maverick",
        "llama-4-scout",
        "llama-3.3-70b-instruct"
      ]
    },
    "mistralai": {
      "models": [
        "ministral-8b",
        "mistral-small-3.1-24b-instruct",
        "mistral-nemo",
        "magistral-small-2506",
        "devstral-medium",
        "devstral-small",
        "codestral-2508",
        "mistral-medium-3.1"
      ]
    },
    "openai": {
      "models": [
        "gpt-4.1",
        "gpt-4.1-nano",
        "gpt-4o-mini",
        "o3",
        "o3-mini",
        "o3-pro",
        "o4-mini",
        "o4-mini-high",
        "gpt-oss-20b",
        "gpt-oss-120b",
        "gpt-5",
        "gpt-5-nano",
        "gpt-5-mini",
        "gpt-5-codex"
      ]
    },
    "qwen": {
      "models": [
        "qwen3-32b",
        "qwen3-235b-a22b",
        "qwen-2.5-coder-32b-instruct",
        "qwen/qwen3-235b-a22b-07-25",
        "qwen3-coder",
        "qwen3-30b-a3b-instruct-2507",
        "qwen3-30b-a3b-thinking-2507",
        "qwen3-max",
        "qwen3-next-80b-a3b-instruct",
        "qwen3-next-80b-a3b-thinking",
        "qwen3-coder-plus",
        "qwen3-coder-flash"
      ]
    },
    "moonshotai": {
      "models": [
        "kimi-k2"
      ]
    },
    "cognitivecomputations": {
      "models": []
    },
    "z-ai": {
      "models": []
    },
    "nvidia": {
      "models": [
        "nemotron-nano-9b-v2"
      ]
    },
    "openrouter": {
      "models": [
        "horizon-alpha",
        "horizon-beta"
      ]
    },
    "alibaba": {
      "models": [
        "tongyi-deepresearch-30b-a3b"
      ]
    },
    "thedrummer": {
      "models": [
        "cydonia-24b-v4.1"
      ]
    }
  }
}