FIX: Fix embeddings to use the old OpenAI tokenizer

Author: romanrizzi

app/models/embedding_definition.rb

@@ -84,7 +84,7 @@ def presets
               dimensions: 2000,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               matryoshka_dimensions: true,
@@ -98,7 +98,7 @@ def presets
               dimensions: 1536,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               matryoshka_dimensions: true,
@@ -112,7 +112,7 @@ def presets
               dimensions: 1536,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               provider_params: {

db/migrate/20250715165701_update_open_ai_embeddings_tokenizer.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+class UpdateOpenAiEmbeddingsTokenizer < ActiveRecord::Migration[7.2]
+  def up
+    execute <<~SQL
+      UPDATE embedding_definitions
+      SET tokenizer_class = 'DiscourseAi::Tokenizer::OpenAiCl100kTokenizer'
+      WHERE url LIKE '%https://api.openai.com/%' AND tokenizer_class <> 'DiscourseAi::Tokenizer::OpenAiCl100kTokenizer'
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

spec/system/embeddings/ai_embedding_definition_spec.rb

@@ -50,7 +50,7 @@
     form.field("provider").select(EmbeddingDefinition::OPEN_AI)
     form.field("url").fill_in("https://api.openai.com/v1/embeddings")
     form.field("api_key").fill_in(api_key)
-    form.field("tokenizer_class").select("DiscourseAi::Tokenizer::OpenAiTokenizer")
+    form.field("tokenizer_class").select("DiscourseAi::Tokenizer::OpenAiCl100kTokenizer")
 
     embed_prefix = "On creation:"
     search_prefix = "On search:"