FEATURE: Add old OpenAI tokenizer to embeddings (#1487)

Author: xfalcox

app/models/embedding_definition.rb

@@ -23,6 +23,7 @@ def tokenizer_names
         DiscourseAi::Tokenizer::GeminiTokenizer,
         DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer,
         DiscourseAi::Tokenizer::OpenAiTokenizer,
+        DiscourseAi::Tokenizer::OpenAiCl100kTokenizer,
         DiscourseAi::Tokenizer::MistralTokenizer,
         DiscourseAi::Tokenizer::QwenTokenizer,
       ].map(&:name)

plugin.rb

@@ -10,7 +10,7 @@
 
 gem "tokenizers", "0.5.4"
 gem "tiktoken_ruby", "0.0.11.1"
-gem "discourse_ai-tokenizers", "0.2.0", require_name: "discourse_ai/tokenizers"
+gem "discourse_ai-tokenizers", "0.3.1", require_name: "discourse_ai/tokenizers"
 gem "ed25519", "1.2.4" #TODO remove this as existing ssl gem should handle this
 
 # we probably want to move all dependencies directly in to the Discourse Gemfile, this