Skip to content

Commit 5b17932

Browse files
pamelafoxpamelafox
committed
Change from LLM inputs to booleans
1 parent 916278e commit 5b17932

File tree

78 files changed

+211
-284
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+211
-284
lines changed

‎.azdo/pipelines/azure-dev.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,9 +121,10 @@ steps:
121121
AZURE_CONTAINER_APPS_WORKLOAD_PROFILE: $(AZURE_CONTAINER_APPS_WORKLOAD_PROFILE)
122122
USE_CHAT_HISTORY_BROWSER: $(USE_CHAT_HISTORY_BROWSER)
123123
USE_MEDIA_DESCRIBER_AZURE_CU: $(USE_MEDIA_DESCRIBER_AZURE_CU)
124-
RAG_LLM_INPUTS_OVERRIDE: $(RAG_LLM_INPUTS_OVERRIDE)
125124
RAG_SEARCH_TEXT_EMBEDDINGS: $(RAG_SEARCH_TEXT_EMBEDDINGS)
126125
RAG_SEARCH_IMAGE_EMBEDDINGS: $(RAG_SEARCH_IMAGE_EMBEDDINGS)
126+
RAG_SEND_TEXT_SOURCES: $(RAG_SEND_TEXT_SOURCES)
127+
RAG_SEND_IMAGE_SOURCES: $(RAG_SEND_IMAGE_SOURCES)
127128
- task: AzureCLI@2
128129
displayName: Deploy Application
129130
inputs:

‎.github/workflows/azure-dev.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,10 @@ jobs:
111111
USE_CHAT_HISTORY_BROWSER: ${{ vars.USE_CHAT_HISTORY_BROWSER }}
112112
USE_MEDIA_DESCRIBER_AZURE_CU: ${{ vars.USE_MEDIA_DESCRIBER_AZURE_CU }}
113113
USE_AI_PROJECT: ${{ vars.USE_AI_PROJECT }}
114-
RAG_LLM_INPUTS_OVERRIDE: ${{ vars.RAG_LLM_INPUTS_OVERRIDE }}
115114
RAG_SEARCH_TEXT_EMBEDDINGS: ${{ vars.RAG_SEARCH_TEXT_EMBEDDINGS }}
116115
RAG_SEARCH_IMAGE_EMBEDDINGS: ${{ vars.RAG_SEARCH_IMAGE_EMBEDDINGS }}
116+
RAG_SEND_TEXT_SOURCES: ${{ vars.RAG_SEND_TEXT_SOURCES }}
117+
RAG_SEND_IMAGE_SOURCES: ${{ vars.RAG_SEND_IMAGE_SOURCES }}
117118
steps:
118119
- name: Checkout
119120
uses: actions/checkout@v4

‎app/backend/app.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,10 @@
7272
CONFIG_MULTIMODAL_ENABLED,
7373
CONFIG_OPENAI_CLIENT,
7474
CONFIG_QUERY_REWRITING_ENABLED,
75-
CONFIG_RAG_LLM_INPUTS_OVERRIDE,
7675
CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS,
7776
CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS,
77+
CONFIG_RAG_SEND_IMAGE_SOURCES,
78+
CONFIG_RAG_SEND_TEXT_SOURCES,
7879
CONFIG_REASONING_EFFORT_ENABLED,
7980
CONFIG_SEARCH_CLIENT,
8081
CONFIG_SEMANTIC_RANKER_DEPLOYED,
@@ -302,9 +303,10 @@ def config():
302303
"showChatHistoryBrowser": current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED],
303304
"showChatHistoryCosmos": current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED],
304305
"showAgenticRetrievalOption": current_app.config[CONFIG_AGENTIC_RETRIEVAL_ENABLED],
305-
"ragLlmInputsOverride": current_app.config[CONFIG_RAG_LLM_INPUTS_OVERRIDE],
306-
"ragSearchImageEmbeddings": current_app.config[CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS],
307306
"ragSearchTextEmbeddings": current_app.config[CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS],
307+
"ragSearchImageEmbeddings": current_app.config[CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS],
308+
"ragSendTextSources": current_app.config[CONFIG_RAG_SEND_TEXT_SOURCES],
309+
"ragSendImageSources": current_app.config[CONFIG_RAG_SEND_IMAGE_SOURCES],
308310
}
309311
)
310312

@@ -454,9 +456,10 @@ async def setup_clients():
454456
AZURE_SPEECH_SERVICE_VOICE = os.getenv("AZURE_SPEECH_SERVICE_VOICE") or "en-US-AndrewMultilingualNeural"
455457

456458
USE_MULTIMODAL = os.getenv("USE_MULTIMODAL", "").lower() == "true"
457-
RAG_LLM_INPUTS_OVERRIDE = os.getenv("RAG_LLM_INPUTS_OVERRIDE", "")
458459
RAG_SEARCH_TEXT_EMBEDDINGS = os.getenv("RAG_SEARCH_TEXT_EMBEDDINGS", "true").lower() == "true"
459460
RAG_SEARCH_IMAGE_EMBEDDINGS = os.getenv("RAG_SEARCH_IMAGE_EMBEDDINGS", "true").lower() == "true"
461+
RAG_SEND_TEXT_SOURCES = os.getenv("RAG_SEND_TEXT_SOURCES", "true").lower() == "true"
462+
RAG_SEND_IMAGE_SOURCES = os.getenv("RAG_SEND_IMAGE_SOURCES", "true").lower() == "true"
460463
USE_USER_UPLOAD = os.getenv("USE_USER_UPLOAD", "").lower() == "true"
461464
ENABLE_LANGUAGE_PICKER = os.getenv("ENABLE_LANGUAGE_PICKER", "").lower() == "true"
462465
USE_SPEECH_INPUT_BROWSER = os.getenv("USE_SPEECH_INPUT_BROWSER", "").lower() == "true"
@@ -660,9 +663,10 @@ async def setup_clients():
660663
current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED] = USE_CHAT_HISTORY_COSMOS
661664
current_app.config[CONFIG_AGENTIC_RETRIEVAL_ENABLED] = USE_AGENTIC_RETRIEVAL
662665
current_app.config[CONFIG_MULTIMODAL_ENABLED] = USE_MULTIMODAL
663-
current_app.config[CONFIG_RAG_LLM_INPUTS_OVERRIDE] = RAG_LLM_INPUTS_OVERRIDE
664666
current_app.config[CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS] = RAG_SEARCH_TEXT_EMBEDDINGS
665667
current_app.config[CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS] = RAG_SEARCH_IMAGE_EMBEDDINGS
668+
current_app.config[CONFIG_RAG_SEND_TEXT_SOURCES] = RAG_SEND_TEXT_SOURCES
669+
current_app.config[CONFIG_RAG_SEND_IMAGE_SOURCES] = RAG_SEND_IMAGE_SOURCES
666670

667671
prompt_manager = PromptyManager()
668672

‎app/backend/approaches/approach.py‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from abc import ABC
22
from collections.abc import AsyncGenerator, Awaitable
33
from dataclasses import dataclass, field
4-
from enum import Enum
54
from typing import Any, Optional, TypedDict, Union, cast
65

76
from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient
@@ -39,12 +38,6 @@
3938
from prepdocslib.embeddings import ImageEmbeddings
4039

4140

42-
class LLMInputType(str, Enum):
43-
TEXT_AND_IMAGES = "textAndImages"
44-
IMAGES = "images"
45-
TEXTS = "texts"
46-
47-
4841
@dataclass
4942
class Document:
5043
id: Optional[str] = None
@@ -204,15 +197,6 @@ def get_storage_client_for_url(self, url: str) -> Optional[Union[ContainerClient
204197
return self.image_datalake_client
205198
return self.image_blob_container_client
206199

207-
def get_default_llm_inputs(self) -> str:
208-
"""
209-
Returns the default LLM inputs based on whether multimodal is enabled
210-
"""
211-
if self.multimodal_enabled:
212-
return LLMInputType.TEXT_AND_IMAGES
213-
else:
214-
return LLMInputType.TEXTS
215-
216200
def build_filter(self, overrides: dict[str, Any], auth_claims: dict[str, Any]) -> Optional[str]:
217201
include_category = overrides.get("include_category")
218202
exclude_category = overrides.get("exclude_category")
@@ -366,7 +350,7 @@ async def get_sources_content(
366350
self,
367351
results: list[Document],
368352
use_semantic_captions: bool,
369-
use_image_sources: bool,
353+
download_image_sources: bool,
370354
user_oid: Optional[str] = None,
371355
) -> tuple[list[str], list[str], list[str]]:
372356
"""
@@ -398,7 +382,7 @@ def nonewlines(s: str) -> str:
398382
else:
399383
text_sources.append(f"{citation}: {nonewlines(doc.content or '')}")
400384

401-
if use_image_sources and hasattr(doc, "images") and doc.images:
385+
if download_image_sources and hasattr(doc, "images") and doc.images:
402386
for img in doc.images:
403387
# Skip if we've already processed this URL
404388
if img["url"] in seen_urls or not img["url"]:

‎app/backend/approaches/chatreadretrieveread.py‎

Lines changed: 17 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
Approach,
2121
DataPoints,
2222
ExtraInfo,
23-
LLMInputType,
2423
ThoughtStep,
2524
)
2625
from approaches.promptmanager import PromptManager
@@ -284,17 +283,10 @@ async def run_search_approach(
284283
minimum_search_score = overrides.get("minimum_search_score", 0.0)
285284
minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
286285
search_index_filter = self.build_filter(overrides, auth_claims)
287-
288-
llm_inputs = overrides.get("llm_inputs")
289-
# Use default values based on multimodal_enabled if not provided in overrides
290-
if llm_inputs is None:
291-
llm_inputs = self.get_default_llm_inputs()
292-
llm_inputs_enum = LLMInputType(llm_inputs) if llm_inputs is not None else None
293-
use_image_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.IMAGES]
294-
use_text_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.TEXTS]
295-
296-
use_image_embeddings = overrides.get("use_image_embeddings", self.multimodal_enabled)
297-
use_text_embeddings = overrides.get("use_text_embeddings", True)
286+
send_text_sources = overrides.get("send_text_sources", True)
287+
send_image_sources = overrides.get("send_image_sources", True)
288+
search_text_embeddings = overrides.get("search_text_embeddings", True)
289+
search_image_embeddings = overrides.get("search_image_embeddings", self.multimodal_enabled)
298290

299291
original_user_query = messages[-1]["content"]
300292
if not isinstance(original_user_query, str):
@@ -329,9 +321,9 @@ async def run_search_approach(
329321

330322
vectors: list[VectorQuery] = []
331323
if use_vector_search:
332-
if use_text_embeddings:
324+
if search_text_embeddings:
333325
vectors.append(await self.compute_text_embedding(query_text))
334-
if use_image_embeddings:
326+
if search_image_embeddings:
335327
vectors.append(await self.compute_multimodal_embedding(query_text))
336328

337329
results = await self.search(
@@ -350,11 +342,11 @@ async def run_search_approach(
350342

351343
# STEP 3: Generate a contextual and content specific answer using the search results and chat history
352344
text_sources, image_sources, citations = await self.get_sources_content(
353-
results, use_semantic_captions, use_image_sources=use_image_sources, user_oid=auth_claims.get("oid")
345+
results, use_semantic_captions, download_image_sources=send_image_sources, user_oid=auth_claims.get("oid")
354346
)
355347

356348
extra_info = ExtraInfo(
357-
DataPoints(text=text_sources if use_text_sources else [], images=image_sources, citations=citations),
349+
DataPoints(text=text_sources if send_text_sources else [], images=image_sources, citations=citations),
358350
thoughts=[
359351
self.format_thought_step_for_chatcompletion(
360352
title="Prompt to generate search query",
@@ -376,8 +368,8 @@ async def run_search_approach(
376368
"filter": search_index_filter,
377369
"use_vector_search": use_vector_search,
378370
"use_text_search": use_text_search,
379-
"use_image_embeddings": use_image_embeddings,
380-
"use_image_sources": use_image_sources,
371+
"search_text_embeddings": search_text_embeddings,
372+
"search_image_embeddings": search_image_embeddings,
381373
},
382374
),
383375
ThoughtStep(
@@ -401,6 +393,8 @@ async def run_agentic_retrieval_approach(
401393
results_merge_strategy = overrides.get("results_merge_strategy", "interleaved")
402394
# 50 is the amount of documents that the reranker can process per query
403395
max_docs_for_reranker = max_subqueries * 50
396+
send_text_sources = overrides.get("send_text_sources", True)
397+
send_image_sources = overrides.get("send_image_sources", True)
404398

405399
response, results = await self.run_agentic_retrieval(
406400
messages=messages,
@@ -413,20 +407,15 @@ async def run_agentic_retrieval_approach(
413407
results_merge_strategy=results_merge_strategy,
414408
)
415409

416-
# Determine if we should use text/image sources based on overrides or defaults
417-
llm_inputs = overrides.get("llm_inputs")
418-
if llm_inputs is None:
419-
llm_inputs = self.get_default_llm_inputs()
420-
llm_inputs_enum = LLMInputType(llm_inputs) if llm_inputs is not None else None
421-
use_image_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.IMAGES]
422-
use_text_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.TEXTS]
423-
424410
text_sources, image_sources, citations = await self.get_sources_content(
425-
results, use_semantic_captions=False, use_image_sources=use_image_sources, user_oid=auth_claims.get("oid")
411+
results,
412+
use_semantic_captions=False,
413+
download_image_sources=send_image_sources,
414+
user_oid=auth_claims.get("oid"),
426415
)
427416

428417
extra_info = ExtraInfo(
429-
DataPoints(text=text_sources if use_text_sources else [], images=image_sources, citations=citations),
418+
DataPoints(text=text_sources if send_text_sources else [], images=image_sources, citations=citations),
430419
thoughts=[
431420
ThoughtStep(
432421
"Use agentic retrieval",

‎app/backend/approaches/retrievethenread.py‎

Lines changed: 13 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
Approach,
1313
DataPoints,
1414
ExtraInfo,
15-
LLMInputType,
1615
ThoughtStep,
1716
)
1817
from approaches.promptmanager import PromptManager
@@ -160,22 +159,15 @@ async def run_search_approach(
160159
minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
161160
filter = self.build_filter(overrides, auth_claims)
162161
q = str(messages[-1]["content"])
163-
164-
llm_inputs = overrides.get("llm_inputs")
165-
# Use default values based on multimodal_enabled if not provided in overrides
166-
if llm_inputs is None:
167-
llm_inputs = self.get_default_llm_inputs()
168-
llm_inputs_enum = LLMInputType(llm_inputs) if llm_inputs is not None else None
169-
use_image_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.IMAGES]
170-
171-
use_image_embeddings = overrides.get("use_image_embeddings", self.multimodal_enabled)
172-
use_text_embeddings = overrides.get("use_text_embeddings", True)
162+
send_image_sources = overrides.get("send_image_sources", True)
163+
search_text_embeddings = overrides.get("search_text_embeddings", True)
164+
search_image_embeddings = overrides.get("search_image_embeddings", self.multimodal_enabled)
173165

174166
vectors: list[VectorQuery] = []
175167
if use_vector_search:
176-
if use_text_embeddings:
168+
if search_text_embeddings:
177169
vectors.append(await self.compute_text_embedding(q))
178-
if use_image_embeddings:
170+
if search_image_embeddings:
179171
vectors.append(await self.compute_multimodal_embedding(q))
180172

181173
results = await self.search(
@@ -193,7 +185,7 @@ async def run_search_approach(
193185
)
194186

195187
text_sources, image_sources, citations = await self.get_sources_content(
196-
results, use_semantic_captions, use_image_sources=use_image_sources, user_oid=auth_claims.get("oid")
188+
results, use_semantic_captions, download_image_sources=send_image_sources, user_oid=auth_claims.get("oid")
197189
)
198190

199191
return ExtraInfo(
@@ -210,8 +202,8 @@ async def run_search_approach(
210202
"filter": filter,
211203
"use_vector_search": use_vector_search,
212204
"use_text_search": use_text_search,
213-
"use_image_embeddings": use_image_embeddings,
214-
"use_image_sources": use_image_sources,
205+
"search_text_embeddings": search_text_embeddings,
206+
"search_image_embeddings": search_image_embeddings,
215207
},
216208
),
217209
ThoughtStep(
@@ -234,6 +226,7 @@ async def run_agentic_retrieval_approach(
234226
results_merge_strategy = overrides.get("results_merge_strategy", "interleaved")
235227
# 50 is the amount of documents that the reranker can process per query
236228
max_docs_for_reranker = max_subqueries * 50
229+
send_image_sources = overrides.get("send_image_sources", True)
237230

238231
response, results = await self.run_agentic_retrieval(
239232
messages,
@@ -246,15 +239,11 @@ async def run_agentic_retrieval_approach(
246239
results_merge_strategy=results_merge_strategy,
247240
)
248241

249-
# Determine if we should use image sources based on overrides or defaults
250-
llm_inputs = overrides.get("llm_inputs")
251-
if llm_inputs is None:
252-
llm_inputs = self.get_default_llm_inputs()
253-
llm_inputs_enum = LLMInputType(llm_inputs) if llm_inputs is not None else None
254-
use_image_sources = llm_inputs_enum in [LLMInputType.TEXT_AND_IMAGES, LLMInputType.IMAGES]
255-
256242
text_sources, image_sources, citations = await self.get_sources_content(
257-
results, use_semantic_captions=False, use_image_sources=use_image_sources, user_oid=auth_claims.get("oid")
243+
results,
244+
use_semantic_captions=False,
245+
download_image_sources=send_image_sources,
246+
user_oid=auth_claims.get("oid"),
258247
)
259248

260249
extra_info = ExtraInfo(

‎app/backend/config.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
CONFIG_COSMOS_HISTORY_CONTAINER = "cosmos_history_container"
3434
CONFIG_COSMOS_HISTORY_VERSION = "cosmos_history_version"
3535
CONFIG_MULTIMODAL_ENABLED = "multimodal_enabled"
36-
CONFIG_RAG_LLM_INPUTS_OVERRIDE = "rag_llm_inputs_override"
3736
CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS = "rag_search_text_embeddings"
3837
CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS = "rag_search_image_embeddings"
38+
CONFIG_RAG_SEND_TEXT_SOURCES = "rag_send_text_sources"
39+
CONFIG_RAG_SEND_IMAGE_SOURCES = "rag_send_image_sources"

‎app/frontend/src/api/models.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,10 @@ export type Config = {
9797
showChatHistoryBrowser: boolean;
9898
showChatHistoryCosmos: boolean;
9999
showAgenticRetrievalOption: boolean;
100-
ragLlmInputsOverride: string;
101-
ragVectorFieldsDefault: string;
100+
ragSearchTextEmbeddings: boolean;
101+
ragSearchImageEmbeddings: boolean;
102+
ragSendTextSources: boolean;
103+
ragSendImageSources: boolean;
102104
};
103105

104106
export type SimpleAPIResponse = {

‎app/frontend/src/components/Settings/Settings.tsx‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -413,14 +413,18 @@ export const Settings = ({
413413
id="sendTextSources"
414414
label={t("labels.llmInputsOptions.texts")}
415415
checked={sendTextSources}
416-
onChange={(_ev, checked) => onChange("send_text_sources", !!checked)}
416+
onChange={(_ev, checked) => {
417+
onChange("sendTextSources", !!checked);
418+
}}
417419
onRenderLabel={props => renderLabel(props, "sendTextSourcesLabel", "sendTextSources", t("helpTexts.llmTextInputs"))}
418420
/>
419421
<Checkbox
420422
id="sendImageSources"
421423
label={t("labels.llmInputsOptions.images")}
422424
checked={sendImageSources}
423-
onChange={(_ev, checked) => onChange("send_image_sources", !!checked)}
425+
onChange={(_ev, checked) => {
426+
onChange("sendImageSources", !!checked);
427+
}}
424428
onRenderLabel={props => renderLabel(props, "sendImageSourcesLabel", "sendImageSources", t("helpTexts.llmImageInputs"))}
425429
/>
426430
</Stack>

‎app/frontend/src/pages/chat/Chat.tsx‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -99,13 +99,11 @@ const Chat = () => {
9999
configApi().then(config => {
100100
setShowMultimodalOptions(config.showMultimodalOptions);
101101
if (config.showMultimodalOptions) {
102-
// Set default LLM inputs based on config override or fallback to TextAndImages
103-
setSendTextSources(true);
104-
setSendImageSources(true);
105-
// Set default vector fields based on config override or fallback to TextAndImageEmbeddings
106-
// Set default vector field settings
107-
setSearchTextEmbeddings(true);
108-
setSearchImageEmbeddings(true);
102+
// Always have at least one source enabled, default to text if none specified
103+
setSendTextSources(config.ragSendTextSources !== undefined ? config.ragSendTextSources : true);
104+
setSendImageSources(config.ragSendImageSources);
105+
setSearchTextEmbeddings(config.ragSearchTextEmbeddings);
106+
setSearchImageEmbeddings(config.ragSearchImageEmbeddings);
109107
}
110108
setUseSemanticRanker(config.showSemanticRankerOption);
111109
setShowSemanticRankerOption(config.showSemanticRankerOption);

0 commit comments

Comments
 (0)