FIX: Fix embeddings to use the old OpenAI tokenizer

Author: romanrizzi

app/models/embedding_definition.rb

@@ -84,7 +84,7 @@ def presets
               dimensions: 2000,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               matryoshka_dimensions: true,
@@ -98,7 +98,7 @@ def presets
               dimensions: 1536,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               matryoshka_dimensions: true,
@@ -112,7 +112,7 @@ def presets
               dimensions: 1536,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               provider_params: {

db/migrate/20250715165701_update_open_ai_embeddings_tokenizer.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+class UpdateOpenAiEmbeddingsTokenizer < ActiveRecord::Migration[7.2]
+  def up
+    execute <<~SQL
+      UPDATE embedding_definitions
+      SET tokenizer_class = 'DiscourseAi::Tokenizer::OpenAiCl100kTokenizer'
+      WHERE url LIKE '%https://api.openai.com/%' AND tokenizer_class <> 'DiscourseAi::Tokenizer::OpenAiCl100kTokenizer'
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end