There was an error while loading. Please reload this page.
1 parent b963edb commit 11ed3feCopy full SHA for 11ed3fe
2 files changed
src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs
@@ -1044,6 +1044,8 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
1044
( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini
1045
1046
// chat
1047
+ ( "gpt-5.6-", ModelEncoding.O200kBase ),
1048
+ ( "gpt-5.5-", ModelEncoding.O200kBase ),
1049
( "gpt-5.4-", ModelEncoding.O200kBase ),
1050
( "gpt-5.3-", ModelEncoding.O200kBase ),
1051
( "gpt-5.2-", ModelEncoding.O200kBase ),
@@ -1075,6 +1077,8 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
1075
1077
{ "o4-mini", ModelEncoding.O200kBase },
1076
1078
1079
1080
+ { "gpt-5.6", ModelEncoding.O200kBase },
1081
+ { "gpt-5.5", ModelEncoding.O200kBase },
1082
{ "gpt-5.4", ModelEncoding.O200kBase },
1083
{ "gpt-5.3", ModelEncoding.O200kBase },
1084
{ "gpt-5.2", ModelEncoding.O200kBase },
test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs
@@ -40,6 +40,8 @@ public class TiktokenTests
40
public static Tokenizer GPT5_2 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.2");
41
public static Tokenizer GPT5_3 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.3");
42
public static Tokenizer GPT5_4 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.4");
43
+ public static Tokenizer GPT5_5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.5");
44
+ public static Tokenizer GPT5_6 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.6");
45
public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4");
46
public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b");
47
@@ -427,6 +429,10 @@ public void TestEncodeR50kBase()
427
429
[InlineData("gpt-5.3-mini")]
428
430
[InlineData("gpt-5.4")]
431
[InlineData("gpt-5.4-nano")]
432
+ [InlineData("gpt-5.5")]
433
+ [InlineData("gpt-5.5-mini")]
434
+ [InlineData("gpt-5.6")]
435
+ [InlineData("gpt-5.6-nano")]
436
[InlineData("chatgpt-4o-")]
437
[InlineData("gpt-4")]
438
[InlineData("gpt-4-")]
@@ -549,6 +555,8 @@ public void TestEncodingNamesNegativeCases()
549
555
[InlineData("gpt-5.2")]
550
556
[InlineData("gpt-5.3")]
551
557
558
559
552
560
[InlineData("o1")]
553
561
[InlineData("o3")]
554
562
[InlineData("o4-mini")]
0 commit comments