Skip to content

Commit 11ed3fe

Browse files
Copilotstephentoub
andauthored
Add GPT-5.5 and GPT-5.6 tokenizer mappings (#7661)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: stephentoub <2642209+stephentoub@users.noreply.github.com>
1 parent b963edb commit 11ed3fe

2 files changed

Lines changed: 12 additions & 0 deletions

File tree

src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1044,6 +1044,8 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
10441044
( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini
10451045

10461046
// chat
1047+
( "gpt-5.6-", ModelEncoding.O200kBase ),
1048+
( "gpt-5.5-", ModelEncoding.O200kBase ),
10471049
( "gpt-5.4-", ModelEncoding.O200kBase ),
10481050
( "gpt-5.3-", ModelEncoding.O200kBase ),
10491051
( "gpt-5.2-", ModelEncoding.O200kBase ),
@@ -1075,6 +1077,8 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
10751077
{ "o4-mini", ModelEncoding.O200kBase },
10761078

10771079
// chat
1080+
{ "gpt-5.6", ModelEncoding.O200kBase },
1081+
{ "gpt-5.5", ModelEncoding.O200kBase },
10781082
{ "gpt-5.4", ModelEncoding.O200kBase },
10791083
{ "gpt-5.3", ModelEncoding.O200kBase },
10801084
{ "gpt-5.2", ModelEncoding.O200kBase },

test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ public class TiktokenTests
4040
public static Tokenizer GPT5_2 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.2");
4141
public static Tokenizer GPT5_3 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.3");
4242
public static Tokenizer GPT5_4 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.4");
43+
public static Tokenizer GPT5_5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.5");
44+
public static Tokenizer GPT5_6 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.6");
4345
public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4");
4446
public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b");
4547

@@ -427,6 +429,10 @@ public void TestEncodeR50kBase()
427429
[InlineData("gpt-5.3-mini")]
428430
[InlineData("gpt-5.4")]
429431
[InlineData("gpt-5.4-nano")]
432+
[InlineData("gpt-5.5")]
433+
[InlineData("gpt-5.5-mini")]
434+
[InlineData("gpt-5.6")]
435+
[InlineData("gpt-5.6-nano")]
430436
[InlineData("chatgpt-4o-")]
431437
[InlineData("gpt-4")]
432438
[InlineData("gpt-4-")]
@@ -549,6 +555,8 @@ public void TestEncodingNamesNegativeCases()
549555
[InlineData("gpt-5.2")]
550556
[InlineData("gpt-5.3")]
551557
[InlineData("gpt-5.4")]
558+
[InlineData("gpt-5.5")]
559+
[InlineData("gpt-5.6")]
552560
[InlineData("o1")]
553561
[InlineData("o3")]
554562
[InlineData("o4-mini")]

0 commit comments

Comments
 (0)