Change text-embedding-3

Author: pamelafox

app/backend/approaches/approach.py

@@ -44,10 +44,9 @@ class Document:
     reranker_score: Optional[float] = None
 
     def serialize_for_results(self) -> dict[str, Any]:
-        return {
+        result_dict = {
             "id": self.id,
             "content": self.content,
-            "embedding": Document.trim_embedding(self.embedding),
             "imageEmbedding": Document.trim_embedding(self.image_embedding),
             "category": self.category,
             "sourcepage": self.sourcepage,
@@ -69,6 +68,8 @@ def serialize_for_results(self) -> dict[str, Any]:
             "score": self.score,
             "reranker_score": self.reranker_score,
         }
+        result_dict[self.embedding_field] = Document.trim_embedding(self.embedding)
+        return result_dict
 
     @classmethod
     def trim_embedding(cls, embedding: Optional[List[float]]) -> Optional[str]:
@@ -102,6 +103,7 @@ def __init__(
         embedding_deployment: Optional[str],  # Not needed for non-Azure OpenAI or for retrieval_mode="text"
         embedding_model: str,
         embedding_dimensions: int,
+        embedding_field: str,
         openai_host: str,
         vision_endpoint: str,
         vision_token_provider: Callable[[], Awaitable[str]],
@@ -115,6 +117,7 @@ def __init__(
         self.embedding_deployment = embedding_deployment
         self.embedding_model = embedding_model
         self.embedding_dimensions = embedding_dimensions
+        self.embedding_field = embedding_field
         self.openai_host = openai_host
         self.vision_endpoint = vision_endpoint
         self.vision_token_provider = vision_token_provider
@@ -178,7 +181,7 @@ async def search(
                     Document(
                         id=document.get("id"),
                         content=document.get("content"),
-                        embedding=document.get("embedding"),
+                        embedding=document.get(self.embedding_field),
                         image_embedding=document.get("imageEmbedding"),
                         category=document.get("category"),
                         sourcepage=document.get("sourcepage"),
@@ -254,7 +257,8 @@ class ExtraArgs(TypedDict, total=False):
             **dimensions_args,
         )
         query_vector = embedding.data[0].embedding
-        return VectorizedQuery(vector=query_vector, k_nearest_neighbors=50, fields="embedding")
+        # TODO: use optimizations from rag time journey 3
+        return VectorizedQuery(vector=query_vector, k_nearest_neighbors=50, fields=self.embedding)
 
     async def compute_image_embedding(self, q: str):
         endpoint = urljoin(self.vision_endpoint, "computervision/retrieval:vectorizeText")

app/backend/approaches/chatreadretrievereadvision.py

@@ -38,6 +38,7 @@ def __init__(
         embedding_deployment: Optional[str],  # Not needed for non-Azure OpenAI or for retrieval_mode="text"
         embedding_model: str,
         embedding_dimensions: int,
+        embedding_field: str,
         sourcepage_field: str,
         content_field: str,
         query_language: str,
@@ -57,6 +58,7 @@ def __init__(
         self.embedding_deployment = embedding_deployment
         self.embedding_model = embedding_model
         self.embedding_dimensions = embedding_dimensions
+        self.embedding_field = embedding_field
         self.sourcepage_field = sourcepage_field
         self.content_field = content_field
         self.query_language = query_language
@@ -86,7 +88,7 @@ async def run_until_final_call(
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
 
-        vector_fields = overrides.get("vector_fields", ["embedding"])
+        vector_fields = overrides.get("vector_fields", [self.embedding_field])
         send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
         send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
 
@@ -121,9 +123,9 @@ async def run_until_final_call(
         if use_vector_search:
             for field in vector_fields:
                 vector = (
-                    await self.compute_text_embedding(query_text)
-                    if field == "embedding"
-                    else await self.compute_image_embedding(query_text)
+                    await self.compute_image_embedding(query_text)
+                    if field.startswith("image")
+                    else await self.compute_text_embedding(query_text)
                 )
                 vectors.append(vector)
 

app/backend/approaches/retrievethenreadvision.py

@@ -32,6 +32,7 @@ def __init__(
         embedding_deployment: Optional[str],  # Not needed for non-Azure OpenAI or for retrieval_mode="text"
         embedding_model: str,
         embedding_dimensions: int,
+        embedding_field: str,
         sourcepage_field: str,
         content_field: str,
         query_language: str,
@@ -47,6 +48,7 @@ def __init__(
         self.embedding_model = embedding_model
         self.embedding_deployment = embedding_deployment
         self.embedding_dimensions = embedding_dimensions
+        self.embedding_field = embedding_field
         self.sourcepage_field = sourcepage_field
         self.content_field = content_field
         self.gpt4v_deployment = gpt4v_deployment
@@ -81,7 +83,7 @@ async def run(
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
 
-        vector_fields = overrides.get("vector_fields", ["embedding"])
+        vector_fields = overrides.get("vector_fields", [self.embedding_field])
         send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
         send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
 
@@ -90,9 +92,9 @@ async def run(
         if use_vector_search:
             for field in vector_fields:
                 vector = (
-                    await self.compute_text_embedding(q)
-                    if field == "embedding"
-                    else await self.compute_image_embedding(q)
+                    await self.compute_image_embedding(q)
+                    if field.startswith("image")
+                    else await self.compute_text_embedding(q)
                 )
                 vectors.append(vector)
 

app/backend/prepdocslib/integratedvectorizerstrategy.py

@@ -60,7 +60,10 @@ def __init__(
         self.category = category
         self.search_info = search_info
 
-    async def create_embedding_skill(self, index_name: str):
+    async def create_embedding_skill(self, index_name: str, embedding_field: str) -> SearchIndexerSkillset:
+        """
+        Create a skillset for the indexer to chunk documents and generate embeddings
+        """
         skillset_name = f"{index_name}-skillset"
 
         split_skill = SplitSkill(
@@ -87,7 +90,7 @@ async def create_embedding_skill(self, index_name: str):
             inputs=[
                 InputFieldMappingEntry(name="text", source="/document/pages/*"),
             ],
-            outputs=[OutputFieldMappingEntry(name="embedding", target_name="vector")],
+            outputs=[OutputFieldMappingEntry(name=embedding_field, target_name="vector")],
         )
 
         index_projection = SearchIndexerIndexProjection(
@@ -98,7 +101,7 @@ async def create_embedding_skill(self, index_name: str):
                     source_context="/document/pages/*",
                     mappings=[
                         InputFieldMappingEntry(name="content", source="/document/pages/*"),
-                        InputFieldMappingEntry(name="embedding", source="/document/pages/*/vector"),
+                        InputFieldMappingEntry(name=embedding_field, source="/document/pages/*/vector"),
                         InputFieldMappingEntry(name="sourcepage", source="/document/metadata_storage_name"),
                     ],
                 ),

app/backend/prepdocslib/searchmanager.py

@@ -55,6 +55,7 @@ def __init__(
         use_acls: bool = False,
         use_int_vectorization: bool = False,
         embeddings: Optional[OpenAIEmbeddings] = None,
+        embedding_field: str = "embedding3",  # can we make this not have a default?
         search_images: bool = False,
     ):
         self.search_info = search_info
@@ -63,7 +64,9 @@ def __init__(
         self.use_int_vectorization = use_int_vectorization
         self.embeddings = embeddings
         # Integrated vectorization uses the ada-002 model with 1536 dimensions
-        self.embedding_dimensions = self.embeddings.open_ai_dimensions if self.embeddings else 1536
+        # TODO: Update integrated vectorization too!
+        self.embedding_dimensions = self.embeddings.open_ai_dimensions if self.embeddings else None
+        self.embedding_field = embedding_field
         self.search_images = search_images
 
     async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] = None):
@@ -93,7 +96,7 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]]
                         analyzer_name=self.search_analyzer_name,
                     ),
                     SearchField(
-                        name="embedding",
+                        name=self.embedding_field,
                         type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
                         hidden=False,
                         searchable=True,
@@ -204,9 +207,7 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]]
                             VectorSearchProfile(
                                 name="embedding_config",
                                 algorithm_configuration_name="hnsw_config",
-                                vectorizer_name=(
-                                    f"{self.search_info.index_name}-vectorizer" if self.use_int_vectorization else None
-                                ),
+                                vectorizer_name=(f"{self.search_info.index_name}-vectorizer"),
                             ),
                         ],
                         vectorizers=vectorizers,
@@ -228,7 +229,24 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]]
                         ),
                     )
                     await search_index_client.create_or_update_index(existing_index)
-
+                # check if embedding field exists
+                if not any(field.name == self.embedding_field for field in existing_index.fields):
+                    logger.info("Adding embedding field to index %s", self.search_info.index_name)
+                    existing_index.fields.append(
+                        SearchField(
+                            name=self.embedding_field,
+                            type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
+                            hidden=False,
+                            searchable=True,
+                            filterable=False,
+                            sortable=False,
+                            facetable=False,
+                            # TODO: use optimizations here
+                            vector_search_dimensions=self.embedding_dimensions,
+                            vector_search_profile_name="embedding_config",
+                        ),
+                    )
+                    await search_index_client.create_or_update_index(existing_index)
                 if existing_index.vector_search is not None and (
                     existing_index.vector_search.vectorizers is None
                     or len(existing_index.vector_search.vectorizers) == 0
@@ -289,7 +307,7 @@ async def update_content(
                         texts=[section.split_page.text for section in batch]
                     )
                     for i, document in enumerate(documents):
-                        document["embedding"] = embeddings[i]
+                        document[self.embedding_field] = embeddings[i]
                 if image_embeddings:
                     for i, (document, section) in enumerate(zip(documents, batch)):
                         document["imageEmbedding"] = image_embeddings[section.split_page.page_num]

infra/main.bicep

@@ -77,10 +77,13 @@ param chatHistoryVersion string = 'cosmosdb-v2'
 // https://learn.microsoft.com/azure/ai-services/openai/concepts/models?tabs=python-secure%2Cstandard%2Cstandard-chat-completions#standard-deployment-model-availability
 @description('Location for the OpenAI resource group')
 @allowed([
+  'australiaeast'
+  'brazilsouth'
   'canadaeast'
   'eastus'
   'eastus2'
   'francecentral'
+  'germanywestcentral'
   'switzerlandnorth'
   'uksouth'
   'japaneast'
@@ -135,7 +138,7 @@ var chatGpt = {
   modelName: !empty(chatGptModelName) ? chatGptModelName : 'gpt-4o-mini'
   deploymentName: !empty(chatGptDeploymentName) ? chatGptDeploymentName : 'gpt-4o-mini'
   deploymentVersion: !empty(chatGptDeploymentVersion) ? chatGptDeploymentVersion : '2024-07-18'
-  deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'Standard'
+  deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments
   deploymentCapacity: chatGptDeploymentCapacity != 0 ? chatGptDeploymentCapacity : 30
 }
 
@@ -148,8 +151,8 @@ param embeddingDimensions int = 0
 var embedding = {
   modelName: !empty(embeddingModelName) ? embeddingModelName : 'text-embedding-3-large'
   deploymentName: !empty(embeddingDeploymentName) ? embeddingDeploymentName : 'text-embedding-3-large'
-  deploymentVersion: !empty(embeddingDeploymentVersion) ? embeddingDeploymentVersion : '1'
-  deploymentSkuName: !empty(embeddingDeploymentSkuName) ? embeddingDeploymentSkuName : 'GlobalStandard'
+  deploymentVersion: !empty(embeddingDeploymentVersion) ? embeddingDeploymentVersion : (embeddingModelName == 'text-embedding-ada-002' ? '2' : '1')
+  deploymentSkuName: !empty(embeddingDeploymentSkuName) ? embeddingDeploymentSkuName : (embeddingModelName == 'text-embedding-ada-002' ? 'Standard' : 'GlobalStandard')
   deploymentCapacity: embeddingDeploymentCapacity != 0 ? embeddingDeploymentCapacity : 30
   dimensions: embeddingDimensions != 0 ? embeddingDimensions : 3072
 }
@@ -163,7 +166,7 @@ var gpt4v = {
   modelName: !empty(gpt4vModelName) ? gpt4vModelName : 'gpt-4o'
   deploymentName: !empty(gpt4vDeploymentName) ? gpt4vDeploymentName : 'gpt-4o'
   deploymentVersion: !empty(gpt4vModelVersion) ? gpt4vModelVersion : '2024-08-06'
-  deploymentSkuName: !empty(gpt4vDeploymentSkuName) ? gpt4vDeploymentSkuName : 'Standard'
+  deploymentSkuName: !empty(gpt4vDeploymentSkuName) ? gpt4vDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments
   deploymentCapacity: gpt4vDeploymentCapacity != 0 ? gpt4vDeploymentCapacity : 10
 }
 
@@ -176,7 +179,7 @@ var eval = {
   modelName: !empty(evalModelName) ? evalModelName : 'gpt-4o'
   deploymentName: !empty(evalDeploymentName) ? evalDeploymentName : 'gpt-4o'
   deploymentVersion: !empty(evalModelVersion) ? evalModelVersion : '2024-08-06'
-  deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'Standard'
+  deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments
   deploymentCapacity: evalDeploymentCapacity != 0 ? evalDeploymentCapacity : 30
 }
 
@@ -1235,6 +1238,7 @@ output AZURE_RESOURCE_GROUP string = resourceGroup.name
 // Shared by all OpenAI deployments
 output OPENAI_HOST string = openAiHost
 output AZURE_OPENAI_EMB_MODEL_NAME string = embedding.modelName
+output AZURE_OPENAI_EMB_DIMENSIONS int = embedding.dimensions
 output AZURE_OPENAI_CHATGPT_MODEL string = chatGpt.modelName
 output AZURE_OPENAI_GPT4V_MODEL string = gpt4v.modelName
 
@@ -1243,9 +1247,17 @@ output AZURE_OPENAI_SERVICE string = isAzureOpenAiHost && deployAzureOpenAi ? op
 output AZURE_OPENAI_API_VERSION string = isAzureOpenAiHost ? azureOpenAiApiVersion : ''
 output AZURE_OPENAI_RESOURCE_GROUP string = isAzureOpenAiHost ? openAiResourceGroup.name : ''
 output AZURE_OPENAI_CHATGPT_DEPLOYMENT string = isAzureOpenAiHost ? chatGpt.deploymentName : ''
+output AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION string = isAzureOpenAiHost ? chatGpt.deploymentVersion : ''
+output AZURE_OPENAI_CHATGPT_DEPLOYMENT_SKU string = isAzureOpenAiHost ? chatGpt.deploymentSkuName : ''
 output AZURE_OPENAI_EMB_DEPLOYMENT string = isAzureOpenAiHost ? embedding.deploymentName : ''
+output AZURE_OPENAI_EMB_DEPLOYMENT_VERSION string = isAzureOpenAiHost ? embedding.deploymentVersion : ''
+output AZURE_OPENAI_EMB_DEPLOYMENT_SKU string = isAzureOpenAiHost ? embedding.deploymentSkuName : ''
 output AZURE_OPENAI_GPT4V_DEPLOYMENT string = isAzureOpenAiHost && useGPT4V ? gpt4v.deploymentName : ''
+output AZURE_OPENAI_GPT4V_DEPLOYMENT_VERSION string = isAzureOpenAiHost && useGPT4V ? gpt4v.deploymentVersion : ''
+output AZURE_OPENAI_GPT4V_DEPLOYMENT_SKU string = isAzureOpenAiHost && useGPT4V ? gpt4v.deploymentSkuName : ''
 output AZURE_OPENAI_EVAL_DEPLOYMENT string = isAzureOpenAiHost && useEval ? eval.deploymentName : ''
+output AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION string = isAzureOpenAiHost && useEval ? eval.deploymentVersion : ''
+output AZURE_OPENAI_EVAL_DEPLOYMENT_SKU string = isAzureOpenAiHost && useEval ? eval.deploymentSkuName : ''
 output AZURE_OPENAI_EVAL_MODEL string = isAzureOpenAiHost && useEval ? eval.modelName : ''
 
 output AZURE_SPEECH_SERVICE_ID string = useSpeechOutputAzure ? speech.outputs.resourceId : ''