From 56ec23bcc3694e330f67ea67a0d0bb8eb9b97c3f Mon Sep 17 00:00:00 2001 From: Rafael Silva Date: Fri, 4 Jul 2025 18:20:44 -0300 Subject: [PATCH 1/2] FEATURE: Add old OpenAI tokenizer to embeddings --- app/models/embedding_definition.rb | 1 + plugin.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/embedding_definition.rb b/app/models/embedding_definition.rb index 8a144f911..23b37ec4a 100644 --- a/app/models/embedding_definition.rb +++ b/app/models/embedding_definition.rb @@ -23,6 +23,7 @@ def tokenizer_names DiscourseAi::Tokenizer::GeminiTokenizer, DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer, DiscourseAi::Tokenizer::OpenAiTokenizer, + DiscourseAi::Tokenizer::OpenAiCl100kTokenizer, DiscourseAi::Tokenizer::MistralTokenizer, DiscourseAi::Tokenizer::QwenTokenizer, ].map(&:name) diff --git a/plugin.rb b/plugin.rb index daf9c10ef..30afc1e7d 100644 --- a/plugin.rb +++ b/plugin.rb @@ -10,7 +10,7 @@ gem "tokenizers", "0.5.4" gem "tiktoken_ruby", "0.0.11.1" -gem "discourse_ai-tokenizers", "0.2.0", require_name: "discourse_ai/tokenizers" +gem "discourse_ai-tokenizers", "0.3.0", require_name: "discourse_ai/tokenizers" gem "ed25519", "1.2.4" #TODO remove this as existing ssl gem should handle this # we probably want to move all dependencies directly in to the Discourse Gemfile, this From 919c08a39bc97b531e5ff25a5a12b592d01dc80f Mon Sep 17 00:00:00 2001 From: Rafael Silva Date: Mon, 7 Jul 2025 14:32:27 -0300 Subject: [PATCH 2/2] bump tokenizers --- plugin.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.rb b/plugin.rb index 30afc1e7d..6758238ff 100644 --- a/plugin.rb +++ b/plugin.rb @@ -10,7 +10,7 @@ gem "tokenizers", "0.5.4" gem "tiktoken_ruby", "0.0.11.1" -gem "discourse_ai-tokenizers", "0.3.0", require_name: "discourse_ai/tokenizers" +gem "discourse_ai-tokenizers", "0.3.1", require_name: "discourse_ai/tokenizers" gem "ed25519", "1.2.4" #TODO remove this as existing ssl gem should handle this # we probably want to move all dependencies directly in to the Discourse Gemfile, this