fix: add missing 'text' and 'layoutText' fields for Azure Search integrated vectorization (#1839)

Co-authored-by: Vamshi-Microsoft <[email protected]>
Co-authored-by: Prajwal-Microsoft <[email protected]>

Author: 3 people

.github/workflows/broken-links-checker.yml

@@ -6,6 +6,9 @@ on:
       - '**/*.md'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   markdown-link-check:
     name: Check Markdown Broken Links
@@ -26,14 +29,14 @@ jobs:
           echo "md_files<<EOF" >> $GITHUB_OUTPUT
           echo "$files" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
-
       - name: Check Broken Links in Added/Modified Files (PR)
         if: github.event_name == 'pull_request' && steps.changed-files.outputs.md_files != ''
         uses: lycheeverse/[email protected]
         with:
           args: >
             --verbose --exclude-mail --no-progress --exclude ^https?://
             ${{ steps.changed-files.outputs.md_files }}
+          failIfEmpty: false
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -44,6 +47,7 @@ jobs:
           args: >
             --verbose --exclude-mail --no-progress --exclude ^https?://
             '**/*.md'
+          failIfEmpty: false
           output: lychee/out.md
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

code/backend/batch/utilities/helpers/azure_search_helper.py

@@ -132,6 +132,25 @@ def create_index(self):
                 filterable=True,
             ),
         ]
+        if self.env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION:
+            logger.info("Adding 'text' field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name=self.env_helper.AZURE_SEARCH_TEXT_COLUMN,
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
+            logger.info("Adding 'layoutText' field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name=self.env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN,
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
 
         if self.env_helper.USE_ADVANCED_IMAGE_PROCESSING:
             logger.info("Adding image_vector field to index")
@@ -274,7 +293,7 @@ def get_conversation_logger(self):
                 embedding_function=self.llm_helper.get_embedding_model().embed_query,
                 fields=fields,
                 user_agent="langchain chatwithyourdata-sa",
-                credential=credential  # Add token credential or send none so it is auto handled by AzureSearch library
+                credential=credential,  # Add token credential or send none so it is auto handled by AzureSearch library
             )
         else:
             return AzureSearch(

code/backend/batch/utilities/helpers/env_helper.py

@@ -76,6 +76,10 @@ def __load_config(self, **kwargs) -> None:
         self.AZURE_SEARCH_SOURCE_COLUMN = os.getenv(
             "AZURE_SEARCH_SOURCE_COLUMN", "source"
         )
+        self.AZURE_SEARCH_TEXT_COLUMN = os.getenv("AZURE_SEARCH_TEXT_COLUMN", "text")
+        self.AZURE_SEARCH_LAYOUT_TEXT_COLUMN = os.getenv(
+            "AZURE_SEARCH_LAYOUT_TEXT_COLUMN", "layoutText"
+        )
         self.AZURE_SEARCH_CHUNK_COLUMN = os.getenv("AZURE_SEARCH_CHUNK_COLUMN", "chunk")
         self.AZURE_SEARCH_OFFSET_COLUMN = os.getenv(
             "AZURE_SEARCH_OFFSET_COLUMN", "offset"
@@ -173,9 +177,7 @@ def __load_config(self, **kwargs) -> None:
             self.AZURE_OPENAI_MODEL_NAME = azure_openai_model_info.get("modelName", "")
         else:
             # Otherwise, fallback to individual environment variables
-            self.AZURE_OPENAI_MODEL = os.getenv(
-                "AZURE_OPENAI_MODEL", "gpt-4.1"
-            )
+            self.AZURE_OPENAI_MODEL = os.getenv("AZURE_OPENAI_MODEL", "gpt-4.1")
             self.AZURE_OPENAI_MODEL_NAME = os.getenv(
                 "AZURE_OPENAI_MODEL_NAME", "gpt-4.1"
             )

code/backend/batch/utilities/integrated_vectorization/azure_search_index.py

@@ -99,6 +99,26 @@ def create_or_update_index(self):
             ),
         ]
 
+        if self.env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION:
+            logger.info("Adding `text` field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name="text",
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
+            logger.info("Adding `layoutText` field for integrated vectorization.")
+            fields.append(
+                SearchableField(
+                    name="layoutText",
+                    type=SearchFieldDataType.String,
+                    filterable=False,
+                    sortable=False,
+                )
+            )
+
         vector_search = self.get_vector_search_config()
 
         semantic_search = self.get_semantic_search_config()

code/create_app.py

@@ -209,6 +209,11 @@ def conversation_with_data(conversation: Request, env_helper: EnvHelper):
                                 env_helper.AZURE_SEARCH_CONTENT_VECTOR_COLUMN
                             ],
                             "title_field": env_helper.AZURE_SEARCH_TITLE_COLUMN or None,
+                            "source_field": env_helper.AZURE_SEARCH_SOURCE_COLUMN
+                            or None,
+                            "text_field": env_helper.AZURE_SEARCH_TEXT_COLUMN or None,
+                            "layoutText_field": env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN
+                            or None,
                             "url_field": env_helper.AZURE_SEARCH_FIELDS_METADATA
                             or None,
                             "filepath_field": (

code/tests/functional/tests/backend_api/README.md

@@ -13,7 +13,7 @@ existing set of tests, rather than creating a new set, as this removes the need
 starting up a new instance of the application on another port.
 
 New environment variables common to all tests can be directly added to the `config`
-dict in [app_config.py](../app_config.py), while variables only needed for one set
+dict in [app_config.py](../../app_config.py), while variables only needed for one set
 of tests can be added to the `app_config` fixture in the respective `conftest.py`
 file, e.g. [./default/conftest.py](./default/conftest.py).
 

code/tests/functional/tests/backend_api/with_byod/test_conversation_flow.py

@@ -147,6 +147,9 @@ def test_post_makes_correct_call_to_azure_openai(
                                     "AZURE_SEARCH_FIELDS_METADATA"
                                 ),
                                 "filepath_field": "filepath",
+                                "source_field": "source",
+                                "text_field": "text",
+                                "layoutText_field": "layoutText",
                             },
                             "filter": app_config.get("AZURE_SEARCH_FILTER"),
                             "in_scope": True,

code/tests/functional/tests/functions/integrated_vectorization/test_integrated_vectorization_resource_creation.py

@@ -99,7 +99,9 @@ def test_integrated_vectorization_datasouce_created(
                 "credentials": {
                     "connectionString": f"DefaultEndpointsProtocol=https;AccountName={app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','accountName')};AccountKey={app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','accountKey')};EndpointSuffix=core.windows.net"
                 },
-                "container": {"name": f"{app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','containerName')}"},
+                "container": {
+                    "name": f"{app_config.get_from_json('AZURE_BLOB_STORAGE_INFO','containerName')}"
+                },
                 "dataDeletionDetectionPolicy": {
                     "@odata.type": "#Microsoft.Azure.Search.NativeBlobSoftDeleteDeletionDetectionPolicy"
                 },
@@ -210,6 +212,26 @@ def test_integrated_vectorization_index_created(
                         "facetable": True,
                         "analyzer": "keyword",
                     },
+                    {
+                        "name": "text",
+                        "type": "Edm.String",
+                        "key": False,
+                        "retrievable": True,
+                        "searchable": True,
+                        "filterable": False,
+                        "sortable": False,
+                        "facetable": False,
+                    },
+                    {
+                        "name": "layoutText",
+                        "type": "Edm.String",
+                        "key": False,
+                        "retrievable": True,
+                        "searchable": True,
+                        "filterable": False,
+                        "sortable": False,
+                        "facetable": False,
+                    },
                 ],
                 "semantic": {
                     "configurations": [

code/tests/test_app.py

@@ -25,6 +25,9 @@
 AZURE_SEARCH_CONTENT_COLUMN = "field1|field2"
 AZURE_SEARCH_CONTENT_VECTOR_COLUMN = "vector-column"
 AZURE_SEARCH_TITLE_COLUMN = "title"
+AZURE_SEARCH_SOURCE_COLUMN = "source"
+AZURE_SEARCH_TEXT_COLUMN = "text"
+AZURE_SEARCH_LAYOUT_TEXT_COLUMN = "layoutText"
 AZURE_SEARCH_FILENAME_COLUMN = "filename"
 AZURE_SEARCH_URL_COLUMN = "metadata"
 AZURE_SEARCH_FILTER = "filter"
@@ -73,6 +76,9 @@ def env_helper_mock():
             AZURE_SEARCH_CONTENT_VECTOR_COLUMN
         )
         env_helper.AZURE_SEARCH_TITLE_COLUMN = AZURE_SEARCH_TITLE_COLUMN
+        env_helper.AZURE_SEARCH_SOURCE_COLUMN = AZURE_SEARCH_SOURCE_COLUMN
+        env_helper.AZURE_SEARCH_TEXT_COLUMN = AZURE_SEARCH_TEXT_COLUMN
+        env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN = AZURE_SEARCH_LAYOUT_TEXT_COLUMN
         env_helper.AZURE_SEARCH_FILENAME_COLUMN = AZURE_SEARCH_FILENAME_COLUMN
         env_helper.AZURE_SEARCH_URL_COLUMN = AZURE_SEARCH_URL_COLUMN
         env_helper.AZURE_SEARCH_FILTER = AZURE_SEARCH_FILTER
@@ -684,6 +690,9 @@ def test_conversation_azure_byod_returns_correct_response_when_streaming_with_da
                                 "title_field": AZURE_SEARCH_TITLE_COLUMN,
                                 "url_field": env_helper_mock.AZURE_SEARCH_FIELDS_METADATA,
                                 "filepath_field": AZURE_SEARCH_FILENAME_COLUMN,
+                                "source_field": AZURE_SEARCH_SOURCE_COLUMN,
+                                "text_field": AZURE_SEARCH_TEXT_COLUMN,
+                                "layoutText_field": AZURE_SEARCH_LAYOUT_TEXT_COLUMN,
                             },
                             "filter": AZURE_SEARCH_FILTER,
                             "in_scope": AZURE_SEARCH_ENABLE_IN_DOMAIN,

code/tests/utilities/helpers/test_azure_search_helper.py

@@ -32,6 +32,8 @@
 AZURE_SEARCH_TITLE_COLUMN = "mock-title"
 AZURE_SEARCH_FIELDS_METADATA = "mock-metadata"
 AZURE_SEARCH_SOURCE_COLUMN = "mock-source"
+AZURE_SEARCH_TEXT_COLUMN = "mock-text"
+AZURE_SEARCH_LAYOUT_TEXT_COLUMN = "mock-layout-text"
 AZURE_SEARCH_CHUNK_COLUMN = "mock-chunk"
 AZURE_SEARCH_OFFSET_COLUMN = "mock-offset"
 AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG = "default"
@@ -78,6 +80,8 @@ def env_helper_mock():
         env_helper.AZURE_SEARCH_TITLE_COLUMN = AZURE_SEARCH_TITLE_COLUMN
         env_helper.AZURE_SEARCH_FIELDS_METADATA = AZURE_SEARCH_FIELDS_METADATA
         env_helper.AZURE_SEARCH_SOURCE_COLUMN = AZURE_SEARCH_SOURCE_COLUMN
+        env_helper.AZURE_SEARCH_TEXT_COLUMN = AZURE_SEARCH_TEXT_COLUMN
+        env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN = AZURE_SEARCH_LAYOUT_TEXT_COLUMN
         env_helper.AZURE_SEARCH_CHUNK_COLUMN = AZURE_SEARCH_CHUNK_COLUMN
         env_helper.AZURE_SEARCH_OFFSET_COLUMN = AZURE_SEARCH_OFFSET_COLUMN
         env_helper.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG = (
@@ -232,6 +236,16 @@ def test_creates_search_index_if_not_exists(
             type=SearchFieldDataType.Int32,
             filterable=True,
         ),
+        SearchableField(
+            name=AZURE_SEARCH_TEXT_COLUMN,
+            type=SearchFieldDataType.String,
+            filterable=False,
+        ),
+        SearchableField(
+            name=AZURE_SEARCH_LAYOUT_TEXT_COLUMN,
+            type=SearchFieldDataType.String,
+            filterable=False,
+        ),
     ]
 
     expected_index = SearchIndex(