Merge from main

Author: pamelafox

app/backend/approaches/approach.py

@@ -352,6 +352,7 @@ async def get_sources_content(
         self,
         results: list[Document],
         use_semantic_captions: bool,
+        include_text_sources: bool,
         download_image_sources: bool,
         user_oid: Optional[str] = None,
     ) -> DataPoints:
@@ -382,10 +383,13 @@ def nonewlines(s: str) -> str:
                 citations.append(citation)
 
             # If semantic captions are used, extract captions; otherwise, use content
-            if use_semantic_captions and doc.captions:
-                text_sources.append(f"{citation}: {nonewlines(' . '.join([cast(str, c.text) for c in doc.captions]))}")
-            else:
-                text_sources.append(f"{citation}: {nonewlines(doc.content or '')}")
+            if include_text_sources:
+                if use_semantic_captions and doc.captions:
+                    text_sources.append(
+                        f"{citation}: {nonewlines(' . '.join([cast(str, c.text) for c in doc.captions]))}"
+                    )
+                else:
+                    text_sources.append(f"{citation}: {nonewlines(doc.content or '')}")
 
             if download_image_sources and hasattr(doc, "images") and doc.images:
                 for img in doc.images:
@@ -397,9 +401,7 @@ def nonewlines(s: str) -> str:
                     if url:
                         image_sources.append(url)
                     citations.append(self.get_image_citation(doc.sourcepage or "", img["url"]))
-        if download_image_sources:
-            return DataPoints(text=text_sources, images=image_sources, citations=citations)
-        return DataPoints(text=text_sources, citations=citations)
+        return DataPoints(text=text_sources, images=image_sources, citations=citations)
 
     def get_citation(self, sourcepage: Optional[str]):
         return sourcepage or ""

app/backend/approaches/chatreadretrieveread.py

@@ -16,7 +16,6 @@
 
 from approaches.approach import (
     Approach,
-    DataPoints,
     ExtraInfo,
     ThoughtStep,
 )
@@ -282,9 +281,11 @@ async def run_search_approach(
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         search_index_filter = self.build_filter(overrides, auth_claims)
         send_text_sources = overrides.get("send_text_sources", True)
-        send_image_sources = overrides.get("send_image_sources", True)
+        send_image_sources = overrides.get("send_image_sources", self.multimodal_enabled) and self.multimodal_enabled
         search_text_embeddings = overrides.get("search_text_embeddings", True)
-        search_image_embeddings = overrides.get("search_image_embeddings", self.multimodal_enabled)
+        search_image_embeddings = (
+            overrides.get("search_image_embeddings", self.multimodal_enabled) and self.multimodal_enabled
+        )
 
         original_user_query = messages[-1]["content"]
         if not isinstance(original_user_query, str):
@@ -340,11 +341,12 @@ async def run_search_approach(
 
         # STEP 3: Generate a contextual and content specific answer using the search results and chat history
         data_points = await self.get_sources_content(
-            results, use_semantic_captions, download_image_sources=send_image_sources, user_oid=auth_claims.get("oid")
+            results,
+            use_semantic_captions,
+            include_text_sources=send_text_sources,
+            download_image_sources=send_image_sources,
+            user_oid=auth_claims.get("oid"),
         )
-        if not send_text_sources:
-            data_points = DataPoints(text=[], images=data_points.images, citations=data_points.citations)
-
         extra_info = ExtraInfo(
             data_points,
             thoughts=[
@@ -390,7 +392,7 @@ async def run_agentic_retrieval_approach(
         top = overrides.get("top", 3)
         results_merge_strategy = overrides.get("results_merge_strategy", "interleaved")
         send_text_sources = overrides.get("send_text_sources", True)
-        send_image_sources = overrides.get("send_image_sources", True)
+        send_image_sources = overrides.get("send_image_sources", self.multimodal_enabled) and self.multimodal_enabled
 
         response, results = await self.run_agentic_retrieval(
             messages=messages,
@@ -404,12 +406,10 @@ async def run_agentic_retrieval_approach(
         data_points = await self.get_sources_content(
             results,
             use_semantic_captions=False,
+            include_text_sources=send_text_sources,
             download_image_sources=send_image_sources,
             user_oid=auth_claims.get("oid"),
         )
-        if not send_text_sources:
-            data_points = DataPoints(text=[], images=data_points.images, citations=data_points.citations)
-
         extra_info = ExtraInfo(
             data_points,
             thoughts=[

app/backend/approaches/retrievethenread.py

@@ -157,9 +157,12 @@ async def run_search_approach(
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
         q = str(messages[-1]["content"])
-        send_image_sources = overrides.get("send_image_sources", True)
+        send_text_sources = overrides.get("send_text_sources", True)
+        send_image_sources = overrides.get("send_image_sources", self.multimodal_enabled) and self.multimodal_enabled
         search_text_embeddings = overrides.get("search_text_embeddings", True)
-        search_image_embeddings = overrides.get("search_image_embeddings", self.multimodal_enabled)
+        search_image_embeddings = (
+            overrides.get("search_image_embeddings", self.multimodal_enabled) and self.multimodal_enabled
+        )
 
         vectors: list[VectorQuery] = []
         if use_vector_search:
@@ -183,7 +186,11 @@ async def run_search_approach(
         )
 
         data_points = await self.get_sources_content(
-            results, use_semantic_captions, download_image_sources=send_image_sources, user_oid=auth_claims.get("oid")
+            results,
+            use_semantic_captions,
+            include_text_sources=send_text_sources,
+            download_image_sources=send_image_sources,
+            user_oid=auth_claims.get("oid"),
         )
 
         return ExtraInfo(
@@ -220,7 +227,8 @@ async def run_agentic_retrieval_approach(
         search_index_filter = self.build_filter(overrides, auth_claims)
         top = overrides.get("top", 3)
         results_merge_strategy = overrides.get("results_merge_strategy", "interleaved")
-        send_image_sources = overrides.get("send_image_sources", True)
+        send_text_sources = overrides.get("send_text_sources", True)
+        send_image_sources = overrides.get("send_image_sources", self.multimodal_enabled) and self.multimodal_enabled
 
         response, results = await self.run_agentic_retrieval(
             messages,
@@ -234,6 +242,7 @@ async def run_agentic_retrieval_approach(
         data_points = await self.get_sources_content(
             results,
             use_semantic_captions=False,
+            include_text_sources=send_text_sources,
             download_image_sources=send_image_sources,
             user_oid=auth_claims.get("oid"),
         )

app/backend/prepdocslib/searchmanager.py

@@ -491,35 +491,33 @@ async def update_content(self, sections: list[Section], url: Optional[str] = Non
 
         async with self.search_info.create_search_client() as search_client:
             for batch_index, batch in enumerate(section_batches):
-                image_fields = {}
-                if self.search_images:
-                    image_fields = {
-                        "images": [
-                            {
-                                "url": image.url,
-                                "description": image.description,
-                                "boundingbox": image.bbox,
-                                "embedding": image.embedding,
-                            }
-                            for section in batch
-                            for image in section.chunk.images
-                        ]
-                    }
-                documents = [
-                    {
+                documents = []
+                for section_index, section in enumerate(batch):
+                    image_fields = {}
+                    if self.search_images:
+                        image_fields = {
+                            "images": [
+                                {
+                                    "url": image.url,
+                                    "description": image.description,
+                                    "boundingbox": image.bbox,
+                                    "embedding": image.embedding,
+                                }
+                                for image in section.chunk.images
+                            ]
+                        }
+                    document = {
                         "id": f"{section.content.filename_to_id()}-page-{section_index + batch_index * MAX_BATCH_SIZE}",
                         "content": section.chunk.text,
                         "category": section.category,
                         "sourcepage": BlobManager.sourcepage_from_file_page(
-                            filename=section.content.filename(),
-                            page=section.chunk.page_num,
+                            filename=section.content.filename(), page=section.chunk.page_num
                         ),
                         "sourcefile": section.content.filename(),
                         **image_fields,
                         **section.content.acls,
                     }
-                    for section_index, section in enumerate(batch)
-                ]
+                    documents.append(document)
                 if url:
                     for document in documents:
                         document["storageUrl"] = url

app/frontend/src/pages/ask/Ask.tsx

@@ -36,13 +36,13 @@ export function Component(): JSX.Element {
     const [useQueryRewriting, setUseQueryRewriting] = useState<boolean>(false);
     const [reasoningEffort, setReasoningEffort] = useState<string>("");
     const [sendTextSources, setSendTextSources] = useState<boolean>(true);
-    const [sendImageSources, setSendImageSources] = useState<boolean>(true);
+    const [sendImageSources, setSendImageSources] = useState<boolean>(false);
     const [includeCategory, setIncludeCategory] = useState<string>("");
 
     const [excludeCategory, setExcludeCategory] = useState<string>("");
     const [question, setQuestion] = useState<string>("");
     const [searchTextEmbeddings, setSearchTextEmbeddings] = useState<boolean>(true);
-    const [searchImageEmbeddings, setSearchImageEmbeddings] = useState<boolean>(true);
+    const [searchImageEmbeddings, setSearchImageEmbeddings] = useState<boolean>(false);
     const [useOidSecurityFilter, setUseOidSecurityFilter] = useState<boolean>(false);
     const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState<boolean>(false);
     const [showMultimodalOptions, setShowMultimodalOptions] = useState<boolean>(false);
@@ -86,12 +86,11 @@ export function Component(): JSX.Element {
         configApi().then(config => {
             setShowMultimodalOptions(config.showMultimodalOptions);
             if (config.showMultimodalOptions) {
-                // Set default LLM inputs based on config override or fallback to Texts
-                setSendTextSources(true);
-                setSendImageSources(true);
-                // Set default vector field settings
-                setSearchTextEmbeddings(true);
-                setSearchImageEmbeddings(true);
+                // Initialize from server config so defaults follow deployment settings
+                setSendTextSources(config.ragSendTextSources !== undefined ? config.ragSendTextSources : true);
+                setSendImageSources(config.ragSendImageSources);
+                setSearchTextEmbeddings(config.ragSearchTextEmbeddings);
+                setSearchImageEmbeddings(config.ragSearchImageEmbeddings);
             }
             setUseSemanticRanker(config.showSemanticRankerOption);
             setShowSemanticRankerOption(config.showSemanticRankerOption);

app/frontend/src/pages/chat/Chat.tsx

@@ -47,11 +47,11 @@ const Chat = () => {
     const [excludeCategory, setExcludeCategory] = useState<string>("");
     const [useSuggestFollowupQuestions, setUseSuggestFollowupQuestions] = useState<boolean>(false);
     const [searchTextEmbeddings, setSearchTextEmbeddings] = useState<boolean>(true);
-    const [searchImageEmbeddings, setSearchImageEmbeddings] = useState<boolean>(true);
+    const [searchImageEmbeddings, setSearchImageEmbeddings] = useState<boolean>(false);
     const [useOidSecurityFilter, setUseOidSecurityFilter] = useState<boolean>(false);
     const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState<boolean>(false);
     const [sendTextSources, setSendTextSources] = useState<boolean>(true);
-    const [sendImageSources, setSendImageSources] = useState<boolean>(true);
+    const [sendImageSources, setSendImageSources] = useState<boolean>(false);
 
     const lastQuestionRef = useRef<string>("");
     const chatMessageStreamEnd = useRef<HTMLDivElement | null>(null);
@@ -98,7 +98,7 @@ const Chat = () => {
         configApi().then(config => {
             setShowMultimodalOptions(config.showMultimodalOptions);
             if (config.showMultimodalOptions) {
-                // Always have at least one source enabled, default to text if none specified
+                // Initialize from server config so defaults match deployment settings
                 setSendTextSources(config.ragSendTextSources !== undefined ? config.ragSendTextSources : true);
                 setSendImageSources(config.ragSendImageSources);
                 setSearchTextEmbeddings(config.ragSearchTextEmbeddings);

tests/e2e.py

@@ -93,14 +93,21 @@ def test_chat(sized_page: Page, live_server_url: str):
 
     # Set up a mock route to the /chat endpoint with streaming results
     def handle(route: Route):
-        # Assert that session_state is specified in the request (None for now)
         try:
             post_data = route.request.post_data_json
-            if post_data and "session_state" in post_data:
-                session_state = post_data["session_state"]
-                assert session_state is None
+            # Assert that session_state is specified (None initially)
+            if "session_state" in post_data:
+                assert post_data["session_state"] is None
+            overrides = post_data["context"]["overrides"]
+            # Assert that the default overrides are correct
+            assert overrides.get("send_text_sources") is True
+            assert overrides.get("send_image_sources") is False
+            assert overrides.get("search_text_embeddings") is True
+            assert overrides.get("search_image_embeddings") is False
+            # retrieval_mode may be explicitly "hybrid" or omitted (interpreted as hybrid)
+            assert overrides.get("retrieval_mode") in ["hybrid", None]
         except Exception as e:
-            print(f"Error in test_chat handler: {e}")
+            print(f"Error in test_chat handler (defaults validation): {e}")
 
         # Read the JSONL from our snapshot results and return as the response
         f = open("tests/snapshots/test_app/test_chat_stream_text/client0/result.jsonlines")