Skip to content

Commit f2128ca

Browse files
teomusatoiuteomusatoiu
committed
update for 4o-mini
1 parent 550f6dd commit f2128ca

File tree

1 file changed

+68
-46
lines changed

1 file changed

+68
-46
lines changed

‎examples/How_to_count_tokens_with_tiktoken.ipynb‎

Lines changed: 68 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,14 @@
2222
"\n",
2323
"| Encoding name | OpenAI models |\n",
2424
"|-------------------------|-----------------------------------------------------|\n",
25+
"| `o200k_base` | `gpt-4o`, `gpt-4o-mini` |\n",
2526
"| `cl100k_base` | `gpt-4`, `gpt-3.5-turbo`, `text-embedding-ada-002`, `text-embedding-3-small`, `text-embedding-3-large` |\n",
2627
"| `p50k_base` | Codex models, `text-davinci-002`, `text-davinci-003`|\n",
2728
"| `r50k_base` (or `gpt2`) | GPT-3 models like `davinci` |\n",
2829
"\n",
2930
"You can retrieve the encoding for a model using `tiktoken.encoding_for_model()` as follows:\n",
3031
"```python\n",
31-
"encoding = tiktoken.encoding_for_model('gpt-3.5-turbo')\n",
32+
"encoding = tiktoken.encoding_for_model('gpt-4o-mini')\n",
3233
"```\n",
3334
"\n",
3435
"Note that `p50k_base` overlaps substantially with `r50k_base`, and for non-code applications, they will usually give the same tokens.\n",
@@ -71,12 +72,27 @@
7172
},
7273
{
7374
"cell_type": "code",
74-
"execution_count": null,
75+
"execution_count": 1,
7576
"metadata": {},
76-
"outputs": [],
77+
"outputs": [
78+
{
79+
"name": "stdout",
80+
"output_type": "stream",
81+
"text": [
82+
"\n",
83+
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
84+
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
85+
"Note: you may need to restart the kernel to use updated packages.\n",
86+
"\n",
87+
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
88+
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
89+
"Note: you may need to restart the kernel to use updated packages.\n"
90+
]
91+
}
92+
],
7793
"source": [
78-
"%pip install --upgrade tiktoken\n",
79-
"%pip install --upgrade openai"
94+
"%pip install --upgrade tiktoken -q\n",
95+
"%pip install --upgrade openai -q"
8096
]
8197
},
8298
{
@@ -89,7 +105,7 @@
89105
},
90106
{
91107
"cell_type": "code",
92-
"execution_count": 1,
108+
"execution_count": 2,
93109
"metadata": {},
94110
"outputs": [],
95111
"source": [
@@ -131,7 +147,7 @@
131147
"metadata": {},
132148
"outputs": [],
133149
"source": [
134-
"encoding = tiktoken.encoding_for_model(\"gpt-3.5-turbo\")"
150+
"encoding = tiktoken.encoding_for_model(\"gpt-4o-mini\")"
135151
]
136152
},
137153
{
@@ -159,7 +175,7 @@
159175
{
160176
"data": {
161177
"text/plain": [
162-
"[83, 1609, 5963, 374, 2294, 0]"
178+
"[83, 8251, 2488, 382, 2212, 0]"
163179
]
164180
},
165181
"execution_count": 5,
@@ -236,7 +252,7 @@
236252
{
237253
"data": {
238254
"text/plain": [
239-
"'tiktoken is great!'"
255+
"'turesറلás!'"
240256
]
241257
},
242258
"execution_count": 8,
@@ -272,7 +288,7 @@
272288
{
273289
"data": {
274290
"text/plain": [
275-
"[b't', b'ik', b'token', b' is', b' great', b'!']"
291+
"[b't', b'ures', b'\\xe0\\xb4\\xb1', b'\\xd9\\x84', b'\\xc3\\xa1s', b'!']"
276292
]
277293
},
278294
"execution_count": 9,
@@ -313,7 +329,7 @@
313329
" # print the example string\n",
314330
" print(f'\\nExample string: \"{example_string}\"')\n",
315331
" # for each encoding, print the # of tokens, the token integers, and the token bytes\n",
316-
" for encoding_name in [\"r50k_base\", \"p50k_base\", \"cl100k_base\"]:\n",
332+
" for encoding_name in [\"r50k_base\", \"p50k_base\", \"cl100k_base\", \"o200k_base\"]:\n",
317333
" encoding = tiktoken.get_encoding(encoding_name)\n",
318334
" token_integers = encoding.encode(example_string)\n",
319335
" num_tokens = len(token_integers)\n",
@@ -347,7 +363,11 @@
347363
"\n",
348364
"cl100k_base: 6 tokens\n",
349365
"token integers: [519, 85342, 34500, 479, 8997, 2191]\n",
350-
"token bytes: [b'ant', b'idis', b'establish', b'ment', b'arian', b'ism']\n"
366+
"token bytes: [b'ant', b'idis', b'establish', b'ment', b'arian', b'ism']\n",
367+
"\n",
368+
"o200k_base: 6 tokens\n",
369+
"token integers: [493, 129901, 376, 160388, 21203, 2367]\n",
370+
"token bytes: [b'ant', b'idis', b'est', b'ablishment', b'arian', b'ism']\n"
351371
]
352372
}
353373
],
@@ -377,6 +397,10 @@
377397
"\n",
378398
"cl100k_base: 7 tokens\n",
379399
"token integers: [17, 489, 220, 17, 284, 220, 19]\n",
400+
"token bytes: [b'2', b' +', b' ', b'2', b' =', b' ', b'4']\n",
401+
"\n",
402+
"o200k_base: 7 tokens\n",
403+
"token integers: [17, 659, 220, 17, 314, 220, 19]\n",
380404
"token bytes: [b'2', b' +', b' ', b'2', b' =', b' ', b'4']\n"
381405
]
382406
}
@@ -407,7 +431,11 @@
407431
"\n",
408432
"cl100k_base: 9 tokens\n",
409433
"token integers: [33334, 45918, 243, 21990, 9080, 33334, 62004, 16556, 78699]\n",
410-
"token bytes: [b'\\xe3\\x81\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f', b'\\xe6\\x97\\xa5', b'\\xe3\\x81\\x8a', b'\\xe3\\x82\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8\\xe3\\x81\\x86']\n"
434+
"token bytes: [b'\\xe3\\x81\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f', b'\\xe6\\x97\\xa5', b'\\xe3\\x81\\x8a', b'\\xe3\\x82\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8\\xe3\\x81\\x86']\n",
435+
"\n",
436+
"o200k_base: 8 tokens\n",
437+
"token integers: [8930, 9697, 243, 128225, 8930, 17693, 4344, 48669]\n",
438+
"token bytes: [b'\\xe3\\x81\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f\\xe6\\x97\\xa5', b'\\xe3\\x81\\x8a', b'\\xe3\\x82\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8\\xe3\\x81\\x86']\n"
411439
]
412440
}
413441
],
@@ -433,36 +461,40 @@
433461
},
434462
{
435463
"cell_type": "code",
436-
"execution_count": 2,
464+
"execution_count": 14,
437465
"metadata": {},
438466
"outputs": [],
439467
"source": [
440-
"def num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0613\"):\n",
468+
"def num_tokens_from_messages(messages, model=\"gpt-4o-mini-2024-07-18\"):\n",
441469
" \"\"\"Return the number of tokens used by a list of messages.\"\"\"\n",
442470
" try:\n",
443471
" encoding = tiktoken.encoding_for_model(model)\n",
444472
" except KeyError:\n",
445473
" print(\"Warning: model not found. Using cl100k_base encoding.\")\n",
446474
" encoding = tiktoken.get_encoding(\"cl100k_base\")\n",
447475
" if model in {\n",
448-
" \"gpt-3.5-turbo-0613\",\n",
449-
" \"gpt-3.5-turbo-16k-0613\",\n",
476+
" \"gpt-3.5-turbo-0125\",\n",
450477
" \"gpt-4-0314\",\n",
451478
" \"gpt-4-32k-0314\",\n",
452479
" \"gpt-4-0613\",\n",
453480
" \"gpt-4-32k-0613\",\n",
481+
" \"gpt-4o-mini-2024-07-18\",\n",
482+
" \"gpt-4o-2024-08-06\"\n",
454483
" }:\n",
455484
" tokens_per_message = 3\n",
456485
" tokens_per_name = 1\n",
457-
" elif model == \"gpt-3.5-turbo-0301\":\n",
458-
" tokens_per_message = 4 # every message follows <|start|>{role/name}\\n{content}<|end|>\\n\n",
459-
" tokens_per_name = -1 # if there's a name, the role is omitted\n",
460486
" elif \"gpt-3.5-turbo\" in model:\n",
461-
" print(\"Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.\")\n",
462-
" return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0613\")\n",
487+
" print(\"Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0125.\")\n",
488+
" return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0125\")\n",
463489
" elif \"gpt-4\" in model:\n",
464490
" print(\"Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.\")\n",
465491
" return num_tokens_from_messages(messages, model=\"gpt-4-0613\")\n",
492+
" elif \"gpt-4o\" in model:\n",
493+
" print(\"Warning: gpt-4o and gpt-4o-mini may update over time. Returning num tokens assuming gpt-4o-2024-08-06.\")\n",
494+
" return num_tokens_from_messages(messages, model=\"ggpt-4o-2024-08-06\")\n",
495+
" elif \"gpt-4o-mini\" in model:\n",
496+
" print(\"Warning: gpt-4o-mini may update over time. Returning num tokens assuming gpt-4o-mini-2024-07-18.\")\n",
497+
" return num_tokens_from_messages(messages, model=\"gpt-4o-mini-2024-07-18\")\n",
466498
" else:\n",
467499
" raise NotImplementedError(\n",
468500
" f\"\"\"num_tokens_from_messages() is not implemented for model {model}.\"\"\"\n",
@@ -480,38 +512,36 @@
480512
},
481513
{
482514
"cell_type": "code",
483-
"execution_count": 4,
515+
"execution_count": 15,
484516
"metadata": {},
485517
"outputs": [
486518
{
487519
"name": "stdout",
488520
"output_type": "stream",
489521
"text": [
490-
"gpt-3.5-turbo-0301\n",
491-
"127 prompt tokens counted by num_tokens_from_messages().\n",
492-
"127 prompt tokens counted by the OpenAI API.\n",
493-
"\n",
494-
"gpt-3.5-turbo-0613\n",
522+
"gpt-3.5-turbo\n",
523+
"Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0125.\n",
495524
"129 prompt tokens counted by num_tokens_from_messages().\n",
496525
"129 prompt tokens counted by the OpenAI API.\n",
497526
"\n",
498-
"gpt-3.5-turbo\n",
499-
"Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.\n",
527+
"gpt-4-0613\n",
500528
"129 prompt tokens counted by num_tokens_from_messages().\n",
501529
"129 prompt tokens counted by the OpenAI API.\n",
502530
"\n",
503-
"gpt-4-0314\n",
531+
"gpt-4\n",
532+
"Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.\n",
504533
"129 prompt tokens counted by num_tokens_from_messages().\n",
505534
"129 prompt tokens counted by the OpenAI API.\n",
506535
"\n",
507-
"gpt-4-0613\n",
536+
"gpt-4o\n",
537+
"Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.\n",
508538
"129 prompt tokens counted by num_tokens_from_messages().\n",
509-
"129 prompt tokens counted by the OpenAI API.\n",
539+
"124 prompt tokens counted by the OpenAI API.\n",
510540
"\n",
511-
"gpt-4\n",
541+
"gpt-4o-mini\n",
512542
"Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.\n",
513543
"129 prompt tokens counted by num_tokens_from_messages().\n",
514-
"129 prompt tokens counted by the OpenAI API.\n",
544+
"124 prompt tokens counted by the OpenAI API.\n",
515545
"\n"
516546
]
517547
}
@@ -556,12 +586,11 @@
556586
"]\n",
557587
"\n",
558588
"for model in [\n",
559-
" \"gpt-3.5-turbo-0301\",\n",
560-
" \"gpt-3.5-turbo-0613\",\n",
561589
" \"gpt-3.5-turbo\",\n",
562-
" \"gpt-4-0314\",\n",
563590
" \"gpt-4-0613\",\n",
564591
" \"gpt-4\",\n",
592+
" \"gpt-4o\",\n",
593+
" \"gpt-4o-mini\"\n",
565594
" ]:\n",
566595
" print(model)\n",
567596
" # example token count from the function defined above\n",
@@ -574,13 +603,6 @@
574603
" print(f'{response.usage.prompt_tokens} prompt tokens counted by the OpenAI API.')\n",
575604
" print()\n"
576605
]
577-
},
578-
{
579-
"cell_type": "code",
580-
"execution_count": null,
581-
"metadata": {},
582-
"outputs": [],
583-
"source": []
584606
}
585607
],
586608
"metadata": {
@@ -599,7 +621,7 @@
599621
"name": "python",
600622
"nbconvert_exporter": "python",
601623
"pygments_lexer": "ipython3",
602-
"version": "3.11.5"
624+
"version": "3.12.1"
603625
},
604626
"vscode": {
605627
"interpreter": {

0 commit comments

Comments
 (0)