fix: Enhance Azure AI Search Integration and Refactor E2E Automation (#1843)

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Roopan-Microsoft <[email protected]>
Co-authored-by: Ajit Padhi <[email protected]>
Co-authored-by: Roopan P M <[email protected]>
Co-authored-by: Pavan-Microsoft <[email protected]>
Co-authored-by: Ross Smith <[email protected]>
Co-authored-by: gpickett <[email protected]>
Co-authored-by: Francia Riesco <[email protected]>
Co-authored-by: Francia Riesco <[email protected]>
Co-authored-by: Prajwal D C <[email protected]>
Co-authored-by: Harmanpreet-Microsoft <[email protected]>
Co-authored-by: UtkarshMishra-Microsoft <[email protected]>
Co-authored-by: Priyanka-Microsoft <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Kiran-Siluveru-Microsoft <[email protected]>
Co-authored-by: Prashant-Microsoft <[email protected]>
Co-authored-by: Rohini-Microsoft <[email protected]>
Co-authored-by: Avijit-Microsoft <[email protected]>
Co-authored-by: RaviKiran-Microsoft <[email protected]>
Co-authored-by: Somesh Joshi <[email protected]>
Co-authored-by: Himanshi Agrawal <[email protected]>
Co-authored-by: pradeepjha-microsoft <[email protected]>
Co-authored-by: Harmanpreet Kaur <[email protected]>
Co-authored-by: Bangarraju-Microsoft <[email protected]>
Co-authored-by: Harsh-Microsoft <[email protected]>
Co-authored-by: Kanchan-Microsoft <[email protected]>
Co-authored-by: Cristopher Coronado <[email protected]>
Co-authored-by: Cristopher Coronado Moreira <[email protected]>
Co-authored-by: Vamshi-Microsoft <[email protected]>
Co-authored-by: Thanusree-Microsoft <[email protected]>
Co-authored-by: Niraj Chaudhari (Persistent Systems Inc) <[email protected]>
Co-authored-by: Rohini-Microsoft <[email protected]>

Author: 29 people

code/backend/batch/utilities/helpers/azure_search_helper.py

@@ -132,6 +132,25 @@ def create_index(self):
                 filterable=True,
             ),
         ]
+        if self.env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION:
+            logger.info("Adding 'text' field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name=self.env_helper.AZURE_SEARCH_TEXT_COLUMN,
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
+            logger.info("Adding 'layoutText' field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name=self.env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN,
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
 
         if self.env_helper.USE_ADVANCED_IMAGE_PROCESSING:
             logger.info("Adding image_vector field to index")
@@ -274,7 +293,7 @@ def get_conversation_logger(self):
                 embedding_function=self.llm_helper.get_embedding_model().embed_query,
                 fields=fields,
                 user_agent="langchain chatwithyourdata-sa",
-                credential=credential  # Add token credential or send none so it is auto handled by AzureSearch library
+                credential=credential,  # Add token credential or send none so it is auto handled by AzureSearch library
             )
         else:
             return AzureSearch(

code/backend/batch/utilities/helpers/env_helper.py

@@ -76,6 +76,10 @@ def __load_config(self, **kwargs) -> None:
         self.AZURE_SEARCH_SOURCE_COLUMN = os.getenv(
             "AZURE_SEARCH_SOURCE_COLUMN", "source"
         )
+        self.AZURE_SEARCH_TEXT_COLUMN = os.getenv("AZURE_SEARCH_TEXT_COLUMN", "text")
+        self.AZURE_SEARCH_LAYOUT_TEXT_COLUMN = os.getenv(
+            "AZURE_SEARCH_LAYOUT_TEXT_COLUMN", "layoutText"
+        )
         self.AZURE_SEARCH_CHUNK_COLUMN = os.getenv("AZURE_SEARCH_CHUNK_COLUMN", "chunk")
         self.AZURE_SEARCH_OFFSET_COLUMN = os.getenv(
             "AZURE_SEARCH_OFFSET_COLUMN", "offset"
@@ -173,9 +177,7 @@ def __load_config(self, **kwargs) -> None:
             self.AZURE_OPENAI_MODEL_NAME = azure_openai_model_info.get("modelName", "")
         else:
             # Otherwise, fallback to individual environment variables
-            self.AZURE_OPENAI_MODEL = os.getenv(
-                "AZURE_OPENAI_MODEL", "gpt-4.1"
-            )
+            self.AZURE_OPENAI_MODEL = os.getenv("AZURE_OPENAI_MODEL", "gpt-4.1")
             self.AZURE_OPENAI_MODEL_NAME = os.getenv(
                 "AZURE_OPENAI_MODEL_NAME", "gpt-4.1"
             )

code/backend/batch/utilities/integrated_vectorization/azure_search_index.py

@@ -99,6 +99,26 @@ def create_or_update_index(self):
             ),
         ]
 
+        if self.env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION:
+            logger.info("Adding `text` field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name="text",
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
+            logger.info("Adding `layoutText` field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name="layoutText",
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
+
         vector_search = self.get_vector_search_config()
 
         semantic_search = self.get_semantic_search_config()

code/create_app.py

@@ -209,6 +209,11 @@ def conversation_with_data(conversation: Request, env_helper: EnvHelper):
                                 env_helper.AZURE_SEARCH_CONTENT_VECTOR_COLUMN
                             ],
                             "title_field": env_helper.AZURE_SEARCH_TITLE_COLUMN or None,
+                            "source_field": env_helper.AZURE_SEARCH_SOURCE_COLUMN
+                            or None,
+                            "text_field": env_helper.AZURE_SEARCH_TEXT_COLUMN or None,
+                            "layoutText_field": env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN
+                            or None,
                             "url_field": env_helper.AZURE_SEARCH_FIELDS_METADATA
                             or None,
                             "filepath_field": (

code/tests/functional/tests/backend_api/with_byod/test_conversation_flow.py

@@ -147,6 +147,9 @@ def test_post_makes_correct_call_to_azure_openai(
                                     "AZURE_SEARCH_FIELDS_METADATA"
                                 ),
                                 "filepath_field": "filepath",
+                                "source_field": "source",
+                                "text_field": "text",
+                                "layoutText_field": "layoutText",
                             },
                             "filter": app_config.get("AZURE_SEARCH_FILTER"),
                             "in_scope": True,

code/tests/functional/tests/functions/integrated_vectorization/test_integrated_vectorization_resource_creation.py

@@ -99,7 +99,9 @@ def test_integrated_vectorization_datasouce_created(
                 "credentials": {
                     "connectionString": f"DefaultEndpointsProtocol=https;AccountName={app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','accountName')};AccountKey={app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','accountKey')};EndpointSuffix=core.windows.net"
                 },
-                "container": {"name": f"{app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','containerName')}"},
+                "container": {
+                    "name": f"{app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','containerName')}"
+                },
                 "dataDeletionDetectionPolicy": {
                     "@odata.type": "#Microsoft.Azure.Search.NativeBlobSoftDeleteDeletionDetectionPolicy"
                 },
@@ -210,6 +212,26 @@ def test_integrated_vectorization_index_created(
                         "facetable": True,
                         "analyzer": "keyword",
                     },
+                    {
+                        "name": "text",
+                        "type": "Edm.String",
+                        "key": False,
+                        "retrievable": True,
+                        "searchable": True,
+                        "filterable": False,
+                        "sortable": False,
+                        "facetable": False,
+                    },
+                    {
+                        "name": "layoutText",
+                        "type": "Edm.String",
+                        "key": False,
+                        "retrievable": True,
+                        "searchable": True,
+                        "filterable": False,
+                        "sortable": False,
+                        "facetable": False,
+                    },
                 ],
                 "semantic": {
                     "configurations": [

code/tests/test_app.py

@@ -25,6 +25,9 @@
 AZURE_SEARCH_CONTENT_COLUMN = "field1|field2"
 AZURE_SEARCH_CONTENT_VECTOR_COLUMN = "vector-column"
 AZURE_SEARCH_TITLE_COLUMN = "title"
+AZURE_SEARCH_SOURCE_COLUMN = "source"
+AZURE_SEARCH_TEXT_COLUMN = "text"
+AZURE_SEARCH_LAYOUT_TEXT_COLUMN = "layoutText"
 AZURE_SEARCH_FILENAME_COLUMN = "filename"
 AZURE_SEARCH_URL_COLUMN = "metadata"
 AZURE_SEARCH_FILTER = "filter"
@@ -73,6 +76,9 @@ def env_helper_mock():
             AZURE_SEARCH_CONTENT_VECTOR_COLUMN
         )
         env_helper.AZURE_SEARCH_TITLE_COLUMN = AZURE_SEARCH_TITLE_COLUMN
+        env_helper.AZURE_SEARCH_SOURCE_COLUMN = AZURE_SEARCH_SOURCE_COLUMN
+        env_helper.AZURE_SEARCH_TEXT_COLUMN = AZURE_SEARCH_TEXT_COLUMN
+        env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN = AZURE_SEARCH_LAYOUT_TEXT_COLUMN
         env_helper.AZURE_SEARCH_FILENAME_COLUMN = AZURE_SEARCH_FILENAME_COLUMN
         env_helper.AZURE_SEARCH_URL_COLUMN = AZURE_SEARCH_URL_COLUMN
         env_helper.AZURE_SEARCH_FILTER = AZURE_SEARCH_FILTER
@@ -684,6 +690,9 @@ def test_conversation_azure_byod_returns_correct_response_when_streaming_with_da
                                 "title_field": AZURE_SEARCH_TITLE_COLUMN,
                                 "url_field": env_helper_mock.AZURE_SEARCH_FIELDS_METADATA,
                                 "filepath_field": AZURE_SEARCH_FILENAME_COLUMN,
+                                "source_field": AZURE_SEARCH_SOURCE_COLUMN,
+                                "text_field": AZURE_SEARCH_TEXT_COLUMN,
+                                "layoutText_field": AZURE_SEARCH_LAYOUT_TEXT_COLUMN,
                             },
                             "filter": AZURE_SEARCH_FILTER,
                             "in_scope": AZURE_SEARCH_ENABLE_IN_DOMAIN,

code/tests/utilities/helpers/test_azure_search_helper.py

@@ -32,6 +32,8 @@
 AZURE_SEARCH_TITLE_COLUMN = "mock-title"
 AZURE_SEARCH_FIELDS_METADATA = "mock-metadata"
 AZURE_SEARCH_SOURCE_COLUMN = "mock-source"
+AZURE_SEARCH_TEXT_COLUMN = "mock-text"
+AZURE_SEARCH_LAYOUT_TEXT_COLUMN = "mock-layout-text"
 AZURE_SEARCH_CHUNK_COLUMN = "mock-chunk"
 AZURE_SEARCH_OFFSET_COLUMN = "mock-offset"
 AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG = "default"
@@ -78,6 +80,8 @@ def env_helper_mock():
         env_helper.AZURE_SEARCH_TITLE_COLUMN = AZURE_SEARCH_TITLE_COLUMN
         env_helper.AZURE_SEARCH_FIELDS_METADATA = AZURE_SEARCH_FIELDS_METADATA
         env_helper.AZURE_SEARCH_SOURCE_COLUMN = AZURE_SEARCH_SOURCE_COLUMN
+        env_helper.AZURE_SEARCH_TEXT_COLUMN = AZURE_SEARCH_TEXT_COLUMN
+        env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN = AZURE_SEARCH_LAYOUT_TEXT_COLUMN
         env_helper.AZURE_SEARCH_CHUNK_COLUMN = AZURE_SEARCH_CHUNK_COLUMN
         env_helper.AZURE_SEARCH_OFFSET_COLUMN = AZURE_SEARCH_OFFSET_COLUMN
         env_helper.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG = (
@@ -232,6 +236,16 @@ def test_creates_search_index_if_not_exists(
             type=SearchFieldDataType.Int32,
             filterable=True,
         ),
+        SearchableField(
+            name=AZURE_SEARCH_TEXT_COLUMN,
+            type=SearchFieldDataType.String,
+            filterable=False,
+        ),
+        SearchableField(
+            name=AZURE_SEARCH_LAYOUT_TEXT_COLUMN,
+            type=SearchFieldDataType.String,
+            filterable=False,
+        ),
     ]
 
     expected_index = SearchIndex(

docs/LOCAL_DEPLOYMENT.md

@@ -186,6 +186,9 @@ Execute the above [shell command](#L81) to run the function locally. You may nee
 |AZURE_SEARCH_FIELDS_ID|id|`AZURE_SEARCH_FIELDS_ID`: Field from your Azure AI Search index that gives a unique idenitfier of the document chunk. `id` if you don't have a specific requirement.|
 |AZURE_SEARCH_FILENAME_COLUMN||`AZURE_SEARCH_FILENAME_COLUMN`: Field from your Azure AI Search index that gives a unique idenitfier of the source of your data to display in the UI.|
 |AZURE_SEARCH_TITLE_COLUMN||Field from your Azure AI Search index that gives a relevant title or header for your data content to display in the UI.|
+|AZURE_SEARCH_SOURCE_COLUMN|source|Field from your Azure AI Search index that identifies the source of your data. `source` if you don't have a specific requirement.|
+|AZURE_SEARCH_TEXT_COLUMN|text|Field from your Azure AI Search index that contains the main text content of your documents. `text` if you don't have a specific requirement.|
+|AZURE_SEARCH_LAYOUT_TEXT_COLUMN|layoutText|Field from your Azure AI Search index that contains the layout-aware text content of your documents. `layoutText` if you don't have a specific requirement.|
 |AZURE_SEARCH_URL_COLUMN||Field from your Azure AI Search index that contains a URL for the document, e.g. an Azure Blob Storage URI. This value is not currently used.|
 |AZURE_SEARCH_FIELDS_TAG|tag|Field from your Azure AI Search index that contains tags for the document. `tag` if you don't have a specific requirement.|
 |AZURE_SEARCH_FIELDS_METADATA|metadata|Field from your Azure AI Search index that contains metadata for the document. `metadata` if you don't have a specific requirement.|

docs/TEAMS_LOCAL_DEPLOYMENT.md

@@ -59,6 +59,9 @@ Or use the [Azure Functions VS Code extension](https://marketplace.visualstudio.
 |AZURE_SEARCH_FIELDS_ID|id|`AZURE_SEARCH_FIELDS_ID`: Field from your Azure AI Search index that gives a unique idenitfier of the document chunk. `id` if you don't have a specific requirement.|
 |AZURE_SEARCH_FILENAME_COLUMN||`AZURE_SEARCH_FILENAME_COLUMN`: Field from your Azure AI Search index that gives a unique idenitfier of the source of your data to display in the UI.|
 |AZURE_SEARCH_TITLE_COLUMN||Field from your Azure AI Search index that gives a relevant title or header for your data content to display in the UI.|
+|AZURE_SEARCH_SOURCE_COLUMN|source|Field from your Azure AI Search index that identifies the source of your data. `source` if you don't have a specific requirement.|
+|AZURE_SEARCH_TEXT_COLUMN|text|Field from your Azure AI Search index that contains the main text content of your documents. `text` if you don't have a specific requirement.|
+|AZURE_SEARCH_LAYOUT_TEXT_COLUMN|layoutText|Field from your Azure AI Search index that contains the layout-aware text content of your documents. `layoutText` if you don't have a specific requirement.|
 |AZURE_SEARCH_URL_COLUMN||Field from your Azure AI Search index that contains a URL for the document, e.g. an Azure Blob Storage URI. This value is not currently used.|
 |AZURE_SEARCH_FIELDS_TAG|tag|Field from your Azure AI Search index that contains tags for the document. `tag` if you don't have a specific requirement.|
 |AZURE_SEARCH_FIELDS_METADATA|metadata|Field from your Azure AI Search index that contains metadata for the document. `metadata` if you don't have a specific requirement.|