diff --git a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs index a460135c22..69f2af814f 100644 --- a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs +++ b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs @@ -1044,6 +1044,8 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo ( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini // chat + ( "gpt-5.6-", ModelEncoding.O200kBase ), + ( "gpt-5.5-", ModelEncoding.O200kBase ), ( "gpt-5.4-", ModelEncoding.O200kBase ), ( "gpt-5.3-", ModelEncoding.O200kBase ), ( "gpt-5.2-", ModelEncoding.O200kBase ), @@ -1075,6 +1077,8 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo { "o4-mini", ModelEncoding.O200kBase }, // chat + { "gpt-5.6", ModelEncoding.O200kBase }, + { "gpt-5.5", ModelEncoding.O200kBase }, { "gpt-5.4", ModelEncoding.O200kBase }, { "gpt-5.3", ModelEncoding.O200kBase }, { "gpt-5.2", ModelEncoding.O200kBase }, diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs index cc3814143b..7af5732509 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs @@ -40,6 +40,8 @@ public class TiktokenTests public static Tokenizer GPT5_2 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.2"); public static Tokenizer GPT5_3 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.3"); public static Tokenizer GPT5_4 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.4"); + public static Tokenizer GPT5_5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.5"); + public static Tokenizer GPT5_6 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.6"); public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4"); public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b"); @@ -427,6 +429,10 @@ public void TestEncodeR50kBase() [InlineData("gpt-5.3-mini")] [InlineData("gpt-5.4")] [InlineData("gpt-5.4-nano")] + [InlineData("gpt-5.5")] + [InlineData("gpt-5.5-mini")] + [InlineData("gpt-5.6")] + [InlineData("gpt-5.6-nano")] [InlineData("chatgpt-4o-")] [InlineData("gpt-4")] [InlineData("gpt-4-")] @@ -549,6 +555,8 @@ public void TestEncodingNamesNegativeCases() [InlineData("gpt-5.2")] [InlineData("gpt-5.3")] [InlineData("gpt-5.4")] + [InlineData("gpt-5.5")] + [InlineData("gpt-5.6")] [InlineData("o1")] [InlineData("o3")] [InlineData("o4-mini")]