Use boolean parameters in the backend as well, for vector fields

Author: pamelafox

.azdo/pipelines/azure-dev.yml

@@ -122,7 +122,8 @@ steps:
       USE_CHAT_HISTORY_BROWSER: $(USE_CHAT_HISTORY_BROWSER)
       USE_MEDIA_DESCRIBER_AZURE_CU: $(USE_MEDIA_DESCRIBER_AZURE_CU)
       RAG_LLM_INPUTS_OVERRIDE: $(RAG_LLM_INPUTS_OVERRIDE)
-      RAG_VECTOR_FIELDS_DEFAULT: $(RAG_VECTOR_FIELDS_DEFAULT)
+      RAG_SEARCH_TEXT_EMBEDDINGS: $(RAG_SEARCH_TEXT_EMBEDDINGS)
+      RAG_SEARCH_IMAGE_EMBEDDINGS: $(RAG_SEARCH_IMAGE_EMBEDDINGS)
   - task: AzureCLI@2
     displayName: Deploy Application
     inputs:

.github/workflows/azure-dev.yml

@@ -112,7 +112,8 @@ jobs:
       USE_MEDIA_DESCRIBER_AZURE_CU: ${{ vars.USE_MEDIA_DESCRIBER_AZURE_CU }}
       USE_AI_PROJECT: ${{ vars.USE_AI_PROJECT }}
       RAG_LLM_INPUTS_OVERRIDE: ${{ vars.RAG_LLM_INPUTS_OVERRIDE }}
-      RAG_VECTOR_FIELDS_DEFAULT: ${{ vars.RAG_VECTOR_FIELDS_DEFAULT }}
+      RAG_SEARCH_TEXT_EMBEDDINGS: ${{ vars.RAG_SEARCH_TEXT_EMBEDDINGS }}
+      RAG_SEARCH_IMAGE_EMBEDDINGS: ${{ vars.RAG_SEARCH_IMAGE_EMBEDDINGS }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

app/backend/app.py

@@ -73,7 +73,8 @@
     CONFIG_OPENAI_CLIENT,
     CONFIG_QUERY_REWRITING_ENABLED,
     CONFIG_RAG_LLM_INPUTS_OVERRIDE,
-    CONFIG_RAG_VECTOR_FIELDS_DEFAULT,
+    CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS,
+    CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS,
     CONFIG_REASONING_EFFORT_ENABLED,
     CONFIG_SEARCH_CLIENT,
     CONFIG_SEMANTIC_RANKER_DEPLOYED,
@@ -302,7 +303,8 @@ def config():
             "showChatHistoryCosmos": current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED],
             "showAgenticRetrievalOption": current_app.config[CONFIG_AGENTIC_RETRIEVAL_ENABLED],
             "ragLlmInputsOverride": current_app.config[CONFIG_RAG_LLM_INPUTS_OVERRIDE],
-            "ragVectorFieldsDefault": current_app.config[CONFIG_RAG_VECTOR_FIELDS_DEFAULT],
+            "ragSearchImageEmbeddings": current_app.config[CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS],
+            "ragSearchTextEmbeddings": current_app.config[CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS],
         }
     )
 
@@ -453,7 +455,8 @@ async def setup_clients():
 
     USE_MULTIMODAL = os.getenv("USE_MULTIMODAL", "").lower() == "true"
     RAG_LLM_INPUTS_OVERRIDE = os.getenv("RAG_LLM_INPUTS_OVERRIDE", "")
-    RAG_VECTOR_FIELDS_DEFAULT = os.getenv("RAG_VECTOR_FIELDS_DEFAULT", "")
+    RAG_SEARCH_TEXT_EMBEDDINGS = os.getenv("RAG_SEARCH_TEXT_EMBEDDINGS", "true").lower() == "true"
+    RAG_SEARCH_IMAGE_EMBEDDINGS = os.getenv("RAG_SEARCH_IMAGE_EMBEDDINGS", "true").lower() == "true"
     USE_USER_UPLOAD = os.getenv("USE_USER_UPLOAD", "").lower() == "true"
     ENABLE_LANGUAGE_PICKER = os.getenv("ENABLE_LANGUAGE_PICKER", "").lower() == "true"
     USE_SPEECH_INPUT_BROWSER = os.getenv("USE_SPEECH_INPUT_BROWSER", "").lower() == "true"
@@ -658,7 +661,8 @@ async def setup_clients():
     current_app.config[CONFIG_AGENTIC_RETRIEVAL_ENABLED] = USE_AGENTIC_RETRIEVAL
     current_app.config[CONFIG_MULTIMODAL_ENABLED] = USE_MULTIMODAL
     current_app.config[CONFIG_RAG_LLM_INPUTS_OVERRIDE] = RAG_LLM_INPUTS_OVERRIDE
-    current_app.config[CONFIG_RAG_VECTOR_FIELDS_DEFAULT] = RAG_VECTOR_FIELDS_DEFAULT
+    current_app.config[CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS] = RAG_SEARCH_TEXT_EMBEDDINGS
+    current_app.config[CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS] = RAG_SEARCH_IMAGE_EMBEDDINGS
 
     prompt_manager = PromptyManager()
 

app/backend/approaches/approach.py

@@ -45,12 +45,6 @@ class LLMInputType(str, Enum):
     TEXTS = "texts"
 
 
-class VectorFieldType(str, Enum):
-    EMBEDDING = "textEmbeddingOnly"
-    IMAGE_EMBEDDING = "imageEmbeddingOnly"
-    TEXT_AND_IMAGE_EMBEDDINGS = "textAndImageEmbeddings"
-
-
 @dataclass
 class Document:
     id: Optional[str] = None
@@ -219,15 +213,6 @@ def get_default_llm_inputs(self) -> str:
         else:
             return LLMInputType.TEXTS
 
-    def get_default_vector_fields(self) -> str:
-        """
-        Returns the default vector fields based on whether multimodal is enabled
-        """
-        if self.multimodal_enabled:
-            return VectorFieldType.TEXT_AND_IMAGE_EMBEDDINGS
-        else:
-            return VectorFieldType.EMBEDDING
-
     def build_filter(self, overrides: dict[str, Any], auth_claims: dict[str, Any]) -> Optional[str]:
         include_category = overrides.get("include_category")
         exclude_category = overrides.get("exclude_category")

app/backend/approaches/chatreadretrieveread.py

@@ -22,7 +22,6 @@
     ExtraInfo,
     LLMInputType,
     ThoughtStep,
-    VectorFieldType,
 )
 from approaches.promptmanager import PromptManager
 from core.authentication import AuthenticationHelper
@@ -287,24 +286,16 @@ async def run_search_approach(
         search_index_filter = self.build_filter(overrides, auth_claims)
 
         llm_inputs = overrides.get("llm_inputs")
-        vector_fields = overrides.get("vector_fields")
-
         # Use default values based on multimodal_enabled if not provided in overrides
         if llm_inputs is None:
             llm_inputs = self.get_default_llm_inputs()
-        if vector_fields is None:
-            vector_fields = self.get_default_vector_fields()
-
         llm_inputs_enum = LLMInputType(llm_inputs) if llm_inputs is not None else None
-        vector_fields_enum = VectorFieldType(vector_fields) if vector_fields is not None else None
-        # Use multimodal/image logic based on enums
-        use_image_embeddings = vector_fields_enum in [
-            VectorFieldType.IMAGE_EMBEDDING,
-            VectorFieldType.TEXT_AND_IMAGE_EMBEDDINGS,
-        ]
         use_image_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.IMAGES]
         use_text_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.TEXTS]
 
+        use_image_embeddings = overrides.get("use_image_embeddings", self.multimodal_enabled)
+        use_text_embeddings = overrides.get("use_text_embeddings", True)
+
         original_user_query = messages[-1]["content"]
         if not isinstance(original_user_query, str):
             raise ValueError("The most recent message content must be a string.")
@@ -338,7 +329,8 @@ async def run_search_approach(
 
         vectors: list[VectorQuery] = []
         if use_vector_search:
-            vectors.append(await self.compute_text_embedding(query_text))
+            if use_text_embeddings:
+                vectors.append(await self.compute_text_embedding(query_text))
             if use_image_embeddings:
                 vectors.append(await self.compute_multimodal_embedding(query_text))
 

app/backend/approaches/retrievethenread.py

@@ -14,7 +14,6 @@
     ExtraInfo,
     LLMInputType,
     ThoughtStep,
-    VectorFieldType,
 )
 from approaches.promptmanager import PromptManager
 from core.authentication import AuthenticationHelper
@@ -163,25 +162,18 @@ async def run_search_approach(
         q = str(messages[-1]["content"])
 
         llm_inputs = overrides.get("llm_inputs")
-        vector_fields = overrides.get("vector_fields")
-
         # Use default values based on multimodal_enabled if not provided in overrides
         if llm_inputs is None:
             llm_inputs = self.get_default_llm_inputs()
-        if vector_fields is None:
-            vector_fields = self.get_default_vector_fields()
-
         llm_inputs_enum = LLMInputType(llm_inputs) if llm_inputs is not None else None
-        vector_fields_enum = VectorFieldType(vector_fields) if vector_fields is not None else None
-        use_image_embeddings = vector_fields_enum in [
-            VectorFieldType.IMAGE_EMBEDDING,
-            VectorFieldType.TEXT_AND_IMAGE_EMBEDDINGS,
-        ]
         use_image_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.IMAGES]
 
+        use_image_embeddings = overrides.get("use_image_embeddings", self.multimodal_enabled)
+        use_text_embeddings = overrides.get("use_text_embeddings", True)
+
         vectors: list[VectorQuery] = []
         if use_vector_search:
-            if vector_fields_enum != VectorFieldType.IMAGE_EMBEDDING:
+            if use_text_embeddings:
                 vectors.append(await self.compute_text_embedding(q))
             if use_image_embeddings:
                 vectors.append(await self.compute_multimodal_embedding(q))

app/backend/config.py

@@ -34,4 +34,5 @@
 CONFIG_COSMOS_HISTORY_VERSION = "cosmos_history_version"
 CONFIG_MULTIMODAL_ENABLED = "multimodal_enabled"
 CONFIG_RAG_LLM_INPUTS_OVERRIDE = "rag_llm_inputs_override"
-CONFIG_RAG_VECTOR_FIELDS_DEFAULT = "rag_vector_fields_default"
+CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS = "rag_search_text_embeddings"
+CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS = "rag_search_image_embeddings"

app/frontend/src/locales/en/translation.json

@@ -160,8 +160,6 @@
         "useSemanticCaptions":
              "Sends semantic captions to the LLM instead of the full search result. A semantic caption is extracted from a search result during the process of semantic ranking.",
         "suggestFollowupQuestions": "Asks the LLM to suggest follow-up questions based on the user's query.",
-        "vectorFields":
-            "Specifies which embedding fields in the Azure AI Search Index will be searched, both the 'Images and text' embeddings, 'Images' only, or 'Text' only.",
         "textEmbeddings": "When selected, search will use embeddings from the text-only embeddings model of extracted text chunks.",
         "imageEmbeddings": "When selected, search will use embeddings from the multimodal embeddings model of extracted images.",
         "llmInputs":

azure.yaml

@@ -123,7 +123,8 @@ pipeline:
       - USE_MEDIA_DESCRIBER_AZURE_CU
       - USE_AI_PROJECT
       - RAG_LLM_INPUTS_OVERRIDE
-      - RAG_VECTOR_FIELDS_DEFAULT
+      - RAG_SEARCH_TEXT_EMBEDDINGS
+      - RAG_SEARCH_IMAGE_EMBEDDINGS
   secrets:
       - AZURE_SERVER_APP_SECRET
       - AZURE_CLIENT_APP_SECRET

docs/multimodal.md

@@ -83,16 +83,26 @@ For more details on how this feature works, read [this blog post](https://techco
 
    You can customize the RAG flow approach with a few additional environment variables.
 
-   To only use the text embeddings for the search step (no image embeddings), run:
+   The following variables can be set to either true or false,
+   to control whether Azure AI Search will use text embeddings, image embeddings, or both:
 
    ```shell
-   azd env set RAG_VECTOR_FIELDS_DEFAULT "textEmbeddingOnly"
+   azd env set RAG_SEARCH_TEXT_EMBEDDINGS true
    ```
 
-   To only send text sources to the chat completion model (no images), run:
+   ```shell
+   azd env set RAG_SEARCH_IMAGE_EMBEDDINGS true
+   ```
+
+   The following variable can be set to either true or false,
+   to control whether the chat completion model will use text inputs, image inputs, or both:
+
+   ```shell
+   azd env set RAG_CHAT_TEXT_INPUTS true
+   ```
 
    ```shell
-   azd env set RAG_LLM_INPUTS_OVERRIDE "texts"
+   azd env set RAG_CHAT_IMAGE_INPUTS true
    ```
 
    You can also modify those settings in the "Developer Settings" in the chat UI,