Fix tests for vision, work on vectorizer

Author: pamelafox

app/backend/prepdocs.py

@@ -62,6 +62,7 @@ async def setup_search_info(
     azure_openai_searchagent_deployment: Union[str, None] = None,
     azure_openai_searchagent_model: Union[str, None] = None,
     search_key: Union[str, None] = None,
+    azure_vision_endpoint: Union[str, None] = None,
 ) -> SearchInfo:
     search_creds: Union[AsyncTokenCredential, AzureKeyCredential] = (
         azure_credential if search_key is None else AzureKeyCredential(search_key)
@@ -79,6 +80,7 @@ async def setup_search_info(
         azure_openai_endpoint=azure_openai_endpoint,
         azure_openai_searchagent_model=azure_openai_searchagent_model,
         azure_openai_searchagent_deployment=azure_openai_searchagent_deployment,
+        azure_vision_endpoint=azure_vision_endpoint,
     )
 
 
@@ -455,6 +457,7 @@ async def main(strategy: Strategy, setup_index: bool = True):
             azure_openai_searchagent_model=os.getenv("AZURE_OPENAI_SEARCHAGENT_MODEL"),
             azure_credential=azd_credential,
             search_key=clean_key_if_exists(args.searchkey),
+            azure_vision_endpoint=os.getenv("AZURE_VISION_ENDPOINT"),
         )
     )
     blob_manager = setup_blob_manager(

app/backend/prepdocslib/searchmanager.py

@@ -4,6 +4,8 @@
 from typing import Optional
 
 from azure.search.documents.indexes.models import (
+    AIServicesVisionParameters,
+    AIServicesVisionVectorizer,
     AzureOpenAIVectorizer,
     AzureOpenAIVectorizerParameters,
     BinaryQuantizationCompression,
@@ -152,9 +154,20 @@ async def create_index(self):
                     name="images_hnsw_config",
                     parameters=HnswParameters(metric="cosine"),
                 )
+
+                # Create the AI Vision vectorizer for image embeddings
+                image_vectorizer = AIServicesVisionVectorizer(
+                    vectorizer_name="images-vision-vectorizer",
+                    ai_services_vision_parameters=AIServicesVisionParameters(
+                        resource_uri=self.search_info.azure_vision_endpoint,
+                        model_version="2023-04-15",
+                    ),
+                )
+
                 image_vector_search_profile = VectorSearchProfile(
                     name="images_embedding_profile",
                     algorithm_configuration_name=image_vector_algorithm.name,
+                    vectorizer_name=image_vectorizer.vectorizer_name,
                 )
                 images_field = SearchField(
                     name="images",
@@ -282,7 +295,8 @@ async def create_index(self):
                         raise ValueError("Image search profile and algorithm must be set")
                     vector_search_profiles.append(image_vector_search_profile)
                     vector_algorithms.append(image_vector_algorithm)
-                    # TODO: Add image vectorizer if needed
+                    # Add image vectorizer to vectorizers list
+                    vectorizers.append(image_vectorizer)
 
                 index = SearchIndex(
                     name=self.search_info.index_name,
@@ -366,6 +380,9 @@ async def create_index(self):
                     if existing_index.vector_search.algorithms is None:
                         existing_index.vector_search.algorithms = []
                     existing_index.vector_search.algorithms.append(image_vector_algorithm)
+                    if existing_index.vector_search.vectorizers is None:
+                        existing_index.vector_search.vectorizers = []
+                    existing_index.vector_search.vectorizers.append(image_vectorizer)
                     await search_index_client.create_or_update_index(existing_index)
 
                 if existing_index.semantic_search:
@@ -408,6 +425,7 @@ async def create_index(self):
                             "Can't add vectorizer to search index %s since no Azure OpenAI embeddings service is defined",
                             self.search_info,
                         )
+
         if self.search_info.use_agentic_retrieval and self.search_info.agent_name:
             await self.create_agent()
 

app/backend/prepdocslib/strategy.py

@@ -27,6 +27,7 @@ def __init__(
         azure_openai_searchagent_model: Optional[str] = None,
         azure_openai_searchagent_deployment: Optional[str] = None,
         azure_openai_endpoint: Optional[str] = None,
+        azure_vision_endpoint: Optional[str] = None,
     ):
         self.endpoint = endpoint
         self.credential = credential
@@ -37,6 +38,7 @@ def __init__(
         self.azure_openai_searchagent_model = azure_openai_searchagent_model
         self.azure_openai_searchagent_deployment = azure_openai_searchagent_deployment
         self.azure_openai_endpoint = azure_openai_endpoint
+        self.azure_vision_endpoint = azure_vision_endpoint
 
     def create_search_client(self) -> SearchClient:
         return SearchClient(endpoint=self.endpoint, index_name=self.index_name, credential=self.credential)

tests/mocks.py

@@ -86,7 +86,7 @@ def __init__(self, text, highlights=None, additional_properties=None):
 class MockAsyncSearchResultsIterator:
     def __init__(self, search_text, vector_queries: Optional[list[VectorQuery]]):
         if search_text == "interest rates" or (
-            vector_queries and any([vector.fields == "images/embeddings" for vector in vector_queries])
+            vector_queries and any([vector.fields == "images/embedding" for vector in vector_queries])
         ):
             self.data = [
                 [

tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json

@@ -15,8 +15,8 @@
                 "props": {
                     "filter": null,
                     "top": 3,
-                    "use_image_embeddings": true,
-                    "use_image_sources": true,
+                    "use_image_embeddings": false,
+                    "use_image_sources": false,
                     "use_query_rewriting": false,
                     "use_semantic_captions": false,
                     "use_semantic_ranker": false,