Allow semantic ranker with vector search (#1701)

* Allow semantic ranker with vector search

* Change variable names

* Remove en-us

Author: pamelafox

app/backend/approaches/approach.py

@@ -133,27 +133,34 @@ async def search(
         query_text: Optional[str],
         filter: Optional[str],
         vectors: List[VectorQuery],
+        use_text_search: bool,
+        use_vector_search: bool,
         use_semantic_ranker: bool,
         use_semantic_captions: bool,
         minimum_search_score: Optional[float],
         minimum_reranker_score: Optional[float],
     ) -> List[Document]:
-        # Use semantic ranker if requested and if retrieval mode is text or hybrid (vectors + text)
-        if use_semantic_ranker and query_text:
+        search_text = query_text if use_text_search else ""
+        search_vectors = vectors if use_vector_search else []
+        if use_semantic_ranker:
             results = await self.search_client.search(
-                search_text=query_text,
+                search_text=search_text,
                 filter=filter,
+                top=top,
+                query_caption="extractive|highlight-false" if use_semantic_captions else None,
+                vector_queries=search_vectors,
                 query_type=QueryType.SEMANTIC,
                 query_language=self.query_language,
                 query_speller=self.query_speller,
                 semantic_configuration_name="default",
-                top=top,
-                query_caption="extractive|highlight-false" if use_semantic_captions else None,
-                vector_queries=vectors,
+                semantic_query=query_text,
             )
         else:
             results = await self.search_client.search(
-                search_text=query_text or "", filter=filter, top=top, vector_queries=vectors
+                search_text=search_text,
+                filter=filter,
+                top=top,
+                vector_queries=search_vectors,
             )
 
         documents = []

app/backend/approaches/chatreadretrieveread.py

@@ -88,15 +88,14 @@ async def run_until_final_call(
         auth_claims: dict[str, Any],
         should_stream: bool = False,
     ) -> tuple[dict[str, Any], Coroutine[Any, Any, Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]]]:
-        has_text = overrides.get("retrieval_mode") in ["text", "hybrid", None]
-        has_vector = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
-        use_semantic_captions = True if overrides.get("semantic_captions") and has_text else False
+        use_text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None]
+        use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
+        use_semantic_ranker = True if overrides.get("semantic_ranker") else False
+        use_semantic_captions = True if overrides.get("semantic_captions") else False
         top = overrides.get("top", 3)
         minimum_search_score = overrides.get("minimum_search_score", 0.0)
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
-
         filter = self.build_filter(overrides, auth_claims)
-        use_semantic_ranker = True if overrides.get("semantic_ranker") and has_text else False
 
         original_user_query = messages[-1]["content"]
         if not isinstance(original_user_query, str):
@@ -151,18 +150,16 @@ async def run_until_final_call(
 
         # If retrieval mode includes vectors, compute an embedding for the query
         vectors: list[VectorQuery] = []
-        if has_vector:
+        if use_vector_search:
             vectors.append(await self.compute_text_embedding(query_text))
 
-        # Only keep the text query if the retrieval mode uses text, otherwise drop it
-        if not has_text:
-            query_text = None
-
         results = await self.search(
             top,
             query_text,
             filter,
             vectors,
+            use_text_search,
+            use_vector_search,
             use_semantic_ranker,
             use_semantic_captions,
             minimum_search_score,
@@ -212,7 +209,8 @@ async def run_until_final_call(
                         "use_semantic_ranker": use_semantic_ranker,
                         "top": top,
                         "filter": filter,
-                        "has_vector": has_vector,
+                        "use_vector_search": use_vector_search,
+                        "use_text_search": use_text_search,
                     },
                 ),
                 ThoughtStep(

app/backend/approaches/chatreadretrievereadvision.py

@@ -89,18 +89,18 @@ async def run_until_final_call(
         auth_claims: dict[str, Any],
         should_stream: bool = False,
     ) -> tuple[dict[str, Any], Coroutine[Any, Any, Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]]]:
-        has_text = overrides.get("retrieval_mode") in ["text", "hybrid", None]
-        has_vector = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
-        vector_fields = overrides.get("vector_fields", ["embedding"])
-        use_semantic_captions = True if overrides.get("semantic_captions") and has_text else False
+        use_text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None]
+        use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
+        use_semantic_ranker = True if overrides.get("semantic_ranker") else False
+        use_semantic_captions = True if overrides.get("semantic_captions") else False
         top = overrides.get("top", 3)
         minimum_search_score = overrides.get("minimum_search_score", 0.0)
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
-        use_semantic_ranker = True if overrides.get("semantic_ranker") and has_text else False
 
-        include_gtpV_text = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
-        include_gtpV_images = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
+        vector_fields = overrides.get("vector_fields", ["embedding"])
+        send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
+        send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
 
         original_user_query = messages[-1]["content"]
         if not isinstance(original_user_query, str):
@@ -136,7 +136,7 @@ async def run_until_final_call(
 
         # If retrieval mode includes vectors, compute an embedding for the query
         vectors = []
-        if has_vector:
+        if use_vector_search:
             for field in vector_fields:
                 vector = (
                     await self.compute_text_embedding(query_text)
@@ -145,15 +145,13 @@ async def run_until_final_call(
                 )
                 vectors.append(vector)
 
-        # Only keep the text query if the retrieval mode uses text, otherwise drop it
-        if not has_text:
-            query_text = None
-
         results = await self.search(
             top,
             query_text,
             filter,
             vectors,
+            use_text_search,
+            use_vector_search,
             use_semantic_ranker,
             use_semantic_captions,
             minimum_search_score,
@@ -173,9 +171,9 @@ async def run_until_final_call(
         user_content: list[ChatCompletionContentPartParam] = [{"text": original_user_query, "type": "text"}]
         image_list: list[ChatCompletionContentPartImageParam] = []
 
-        if include_gtpV_text:
+        if send_text_to_gptvision:
             user_content.append({"text": "\n\nSources:\n" + content, "type": "text"})
-        if include_gtpV_images:
+        if send_images_to_gptvision:
             for result in results:
                 url = await fetch_image(self.blob_container_client, result)
                 if url:
@@ -217,6 +215,7 @@ async def run_until_final_call(
                         "top": top,
                         "filter": filter,
                         "vector_fields": vector_fields,
+                        "use_text_search": use_text_search,
                     },
                 ),
                 ThoughtStep(

app/backend/approaches/retrievethenread.py

@@ -80,28 +80,27 @@ async def run(
             raise ValueError("The most recent message content must be a string.")
         overrides = context.get("overrides", {})
         auth_claims = context.get("auth_claims", {})
-        has_text = overrides.get("retrieval_mode") in ["text", "hybrid", None]
-        has_vector = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
-        use_semantic_ranker = overrides.get("semantic_ranker") and has_text
-
-        use_semantic_captions = True if overrides.get("semantic_captions") and has_text else False
+        use_text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None]
+        use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
+        use_semantic_ranker = True if overrides.get("semantic_ranker") else False
+        use_semantic_captions = True if overrides.get("semantic_captions") else False
         top = overrides.get("top", 3)
         minimum_search_score = overrides.get("minimum_search_score", 0.0)
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
+
         # If retrieval mode includes vectors, compute an embedding for the query
         vectors: list[VectorQuery] = []
-        if has_vector:
+        if use_vector_search:
             vectors.append(await self.compute_text_embedding(q))
 
-        # Only keep the text query if the retrieval mode uses text, otherwise drop it
-        query_text = q if has_text else None
-
         results = await self.search(
             top,
-            query_text,
+            q,
             filter,
             vectors,
+            use_text_search,
+            use_vector_search,
             use_semantic_ranker,
             use_semantic_captions,
             minimum_search_score,
@@ -141,13 +140,14 @@ async def run(
             "thoughts": [
                 ThoughtStep(
                     "Search using user query",
-                    query_text,
+                    q,
                     {
                         "use_semantic_captions": use_semantic_captions,
                         "use_semantic_ranker": use_semantic_ranker,
                         "top": top,
                         "filter": filter,
-                        "has_vector": has_vector,
+                        "use_vector_search": use_vector_search,
+                        "use_text_search": use_text_search,
                     },
                 ),
                 ThoughtStep(

app/backend/approaches/retrievethenreadvision.py

@@ -81,24 +81,22 @@ async def run(
 
         overrides = context.get("overrides", {})
         auth_claims = context.get("auth_claims", {})
-        has_text = overrides.get("retrieval_mode") in ["text", "hybrid", None]
-        has_vector = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
-        vector_fields = overrides.get("vector_fields", ["embedding"])
-
-        include_gtpV_text = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
-        include_gtpV_images = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
-
-        use_semantic_captions = True if overrides.get("semantic_captions") and has_text else False
+        use_text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None]
+        use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
+        use_semantic_ranker = True if overrides.get("semantic_ranker") else False
+        use_semantic_captions = True if overrides.get("semantic_captions") else False
         top = overrides.get("top", 3)
         minimum_search_score = overrides.get("minimum_search_score", 0.0)
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
-        use_semantic_ranker = overrides.get("semantic_ranker") and has_text
 
-        # If retrieval mode includes vectors, compute an embedding for the query
+        vector_fields = overrides.get("vector_fields", ["embedding"])
+        send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
+        send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
 
+        # If retrieval mode includes vectors, compute an embedding for the query
         vectors = []
-        if has_vector:
+        if use_vector_search:
             for field in vector_fields:
                 vector = (
                     await self.compute_text_embedding(q)
@@ -107,14 +105,13 @@ async def run(
                 )
                 vectors.append(vector)
 
-        # Only keep the text query if the retrieval mode uses text, otherwise drop it
-        query_text = q if has_text else None
-
         results = await self.search(
             top,
-            query_text,
+            q,
             filter,
             vectors,
+            use_text_search,
+            use_vector_search,
             use_semantic_ranker,
             use_semantic_captions,
             minimum_search_score,
@@ -127,10 +124,10 @@ async def run(
         # Process results
         sources_content = self.get_sources_content(results, use_semantic_captions, use_image_citation=True)
 
-        if include_gtpV_text:
+        if send_text_to_gptvision:
             content = "\n".join(sources_content)
             user_content.append({"text": content, "type": "text"})
-        if include_gtpV_images:
+        if send_images_to_gptvision:
             for result in results:
                 url = await fetch_image(self.blob_container_client, result)
                 if url:
@@ -164,13 +161,15 @@ async def run(
             "thoughts": [
                 ThoughtStep(
                     "Search using user query",
-                    query_text,
+                    q,
                     {
                         "use_semantic_captions": use_semantic_captions,
                         "use_semantic_ranker": use_semantic_ranker,
                         "top": top,
                         "filter": filter,
                         "vector_fields": vector_fields,
+                        "use_vector_search": use_vector_search,
+                        "use_text_search": use_text_search,
                     },
                 ),
                 ThoughtStep(

docs/gpt4v.md

@@ -1,38 +1,33 @@
-# Using GPT-4 Turbo with Vision
+# Using GPT vision model with RAG approach
 
-This repository now includes an example of integrating GPT-4 Turbo with Vision with Azure AI Search. This feature enables indexing and searching images and graphs, such as financial documents, in addition to text-based content.
+This repository now includes an example of integrating a GPT Vision model with Azure AI Search. This feature enables indexing and searching images and graphs, such as financial documents, in addition to text-based content, and then sending the retrieved content to the GPT model for response generation.
 
 ## Feature Overview
 
 - **Document Handling:** Source documents are split into pages and saved as PNG files in blob storage. Each file's name and page number are embedded for reference.
 - **Data Extraction:** Text data is extracted using OCR.
 - **Data Indexing:** Text and image embeddings, generated using Azure AI Vision ([Azure AI Vision Embeddings](https://learn.microsoft.com/azure/ai-services/computer-vision/how-to/image-retrieval)), are indexed in Azure AI Search along with the raw text.
-- **Search and Response:** Searches can be conducted using vectors or hybrid methods. Responses are generated by GPT-4 Turbo with Vision based on the retrieved content.
+- **Search and Response:** Searches can be conducted using vectors or hybrid methods. Responses are generated by GPT vision model based on the retrieved content.
 
 ## Getting Started
 
 ### Prerequisites
 
 - Create a [Computer Vision account in Azure Portal first](https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesComputerVision), so that you can agree to the Responsible AI terms for that resource. You can delete that account after agreeing.
-- The ability to deploy a GPT-4 Turbo with Vision model in the [supported regions](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-preview-model-availability). If you're not sure, try to create a deployment from your Azure OpenAI deployments page. You should be able to select:
-
-   | Model | Version |
-   |--|--|
-   |`gpt-4`|`vision-preview`|
-
+- The ability to deploy a gpt-4o model in the [supported regions](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability). If you're not sure, try to create a gpt-4o deployment from your Azure OpenAI deployments page.
 - Ensure that you can deploy the Azure OpenAI resource group in [a region where all required components are available](https://learn.microsoft.com/azure/cognitive-services/openai/concepts/models#model-summary-table-and-region-availability):
   - Azure OpenAI models
     - gpt-35-turbo
     - text-embedding-ada-002
-    - gpt-4v
+    - gpt-4o
   - [Azure AI Vision](https://learn.microsoft.com/azure/ai-services/computer-vision/)
 
 ### Setup and Usage
 
 1. **Update repository:**
    Pull the latest changes.
 
-2. **Enable GPT-4 Turbo with Vision:**
+2. **Enable GPT vision approach:**
 
    First, make sure you do *not* have integrated vectorization enabled, since that is currently incompatible:
 

tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json

@@ -10,10 +10,11 @@
                 "description": "What is the capital of France?",
                 "props": {
                     "filter": null,
-                    "has_vector": true,
                     "top": 3,
                     "use_semantic_captions": false,
-                    "use_semantic_ranker": null
+                    "use_semantic_ranker": false,
+                    "use_text_search": true,
+                    "use_vector_search": true
                 },
                 "title": "Search using user query"
             },

tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json

@@ -10,10 +10,11 @@
                 "description": "What is the capital of France?",
                 "props": {
                     "filter": null,
-                    "has_vector": true,
                     "top": 3,
                     "use_semantic_captions": false,
-                    "use_semantic_ranker": null
+                    "use_semantic_ranker": false,
+                    "use_text_search": true,
+                    "use_vector_search": true
                 },
                 "title": "Search using user query"
             },

tests/snapshots/test_app/test_ask_rtr_text/client0/result.json

@@ -10,10 +10,11 @@
                 "description": "What is the capital of France?",
                 "props": {
                     "filter": null,
-                    "has_vector": false,
                     "top": 3,
                     "use_semantic_captions": false,
-                    "use_semantic_ranker": null
+                    "use_semantic_ranker": false,
+                    "use_text_search": true,
+                    "use_vector_search": false
                 },
                 "title": "Search using user query"
             },