Change frontend for LLM inputs

Author: pamelafox

app/frontend/src/api/models.ts

@@ -4,12 +4,6 @@ export const enum RetrievalMode {
     Text = "text"
 }
 
-export const enum LLMInputs {
-    TextAndImages = "textAndImages",
-    Images = "images",
-    Texts = "texts"
-}
-
 export type ChatAppRequestOverrides = {
     retrieval_mode?: RetrievalMode;
     semantic_ranker?: boolean;
@@ -31,7 +25,8 @@ export type ChatAppRequestOverrides = {
     suggest_followup_questions?: boolean;
     use_oid_security_filter?: boolean;
     use_groups_security_filter?: boolean;
-    llm_inputs: LLMInputs;
+    send_text_sources: boolean;
+    send_image_sources: boolean;
     search_text_embeddings: boolean;
     search_image_embeddings: boolean;
     language: string;

app/frontend/src/components/Settings/Settings.module.css

@@ -53,3 +53,14 @@
     font-weight: bold;
     color: #fff;
 }
+
+.fieldset {
+    border: none;
+    padding: 0;
+}
+
+.legend {
+    font-size: 14px;
+    margin-bottom: 5px;
+    padding: 0;
+}

app/frontend/src/components/Settings/Settings.tsx

@@ -1,9 +1,9 @@
 import { useId } from "@fluentui/react-hooks";
 import { useTranslation } from "react-i18next";
-import { TextField, ITextFieldProps, Checkbox, ICheckboxProps, Dropdown, IDropdownProps, IDropdownOption } from "@fluentui/react";
+import { TextField, ITextFieldProps, Checkbox, ICheckboxProps, Dropdown, IDropdownProps, IDropdownOption, Stack } from "@fluentui/react";
 import { HelpCallout } from "../HelpCallout";
 import { VectorSettings } from "../VectorSettings";
-import { RetrievalMode, LLMInputs } from "../../api";
+import { RetrievalMode } from "../../api";
 import styles from "./Settings.module.css";
 
 // Add type for onRenderLabel
@@ -25,7 +25,8 @@ export interface SettingsProps {
     excludeCategory: string;
     includeCategory: string;
     retrievalMode: RetrievalMode;
-    llmInputs: LLMInputs;
+    sendTextSources: boolean;
+    sendImageSources: boolean;
     searchTextEmbeddings: boolean;
     searchImageEmbeddings: boolean;
     showSemanticRankerOption: boolean;
@@ -68,7 +69,8 @@ export const Settings = ({
     retrievalMode,
     searchTextEmbeddings,
     searchImageEmbeddings,
-    llmInputs,
+    sendTextSources,
+    sendImageSources,
     showSemanticRankerOption,
     showQueryRewritingOption,
     showReasoningEffortOption,
@@ -402,20 +404,27 @@ export const Settings = ({
                 aria-labelledby={seedId}
                 onRenderLabel={props => renderLabel(props, seedId, seedFieldId, t("helpTexts.seed"))}
             />
+
             {showMultimodalOptions && !useAgenticRetrieval && (
-                <Dropdown
-                    id="llmInputsDropdown"
-                    className={styles.settingsSeparator}
-                    label={t("labels.llmInputs")}
-                    selectedKey={llmInputs}
-                    onChange={(_ev?: React.FormEvent<HTMLElement | HTMLInputElement>, option?: IDropdownOption) => onChange("llmInputs", option?.key)}
-                    options={[
-                        { key: LLMInputs.Texts, text: t("labels.llmInputsOptions.texts") },
-                        { key: LLMInputs.Images, text: t("labels.llmInputsOptions.images") },
-                        { key: LLMInputs.TextAndImages, text: t("labels.llmInputsOptions.textAndImages") }
-                    ]}
-                    onRenderLabel={props => renderLabel(props, "llmInputsDropdownLabel", "llmInputsDropdown", t("helpTexts.llmInputs"))}
-                />
+                <fieldset className={styles.fieldset + " " + styles.settingsSeparator}>
+                    <legend className={styles.legend}>{t("labels.llmInputs")}</legend>
+                    <Stack tokens={{ childrenGap: 8 }}>
+                        <Checkbox
+                            id="sendTextSources"
+                            label={t("labels.llmInputsOptions.texts")}
+                            checked={sendTextSources}
+                            onChange={(_ev, checked) => onChange("send_text_sources", !!checked)}
+                            onRenderLabel={props => renderLabel(props, "sendTextSourcesLabel", "sendTextSources", t("helpTexts.llmTextInputs"))}
+                        />
+                        <Checkbox
+                            id="sendImageSources"
+                            label={t("labels.llmInputsOptions.images")}
+                            checked={sendImageSources}
+                            onChange={(_ev, checked) => onChange("send_image_sources", !!checked)}
+                            onRenderLabel={props => renderLabel(props, "sendImageSourcesLabel", "sendImageSources", t("helpTexts.llmImageInputs"))}
+                        />
+                    </Stack>
+                </fieldset>
             )}
         </div>
     );

app/frontend/src/components/TokenClaimsDisplay/TokenClaimsDisplay.tsx

@@ -83,7 +83,7 @@ export const TokenClaimsDisplay = () => {
     ];
 
     return (
-        <div>
+        <div style={{ marginTop: "20px" }}>
             <Label>ID Token Claims</Label>
             <DataGrid items={items} columns={columns} sortable getRowId={item => item.name}>
                 <DataGridHeader>

app/frontend/src/components/VectorSettings/VectorSettings.module.css

@@ -5,7 +5,6 @@
 .fieldset {
     border: none;
     padding: 0;
-    margin: 0;
 }
 
 .legend {

app/frontend/src/components/VectorSettings/VectorSettings.tsx

@@ -99,7 +99,6 @@ export const VectorSettings = ({
                     <Stack tokens={{ childrenGap: 8 }}>
                         <Checkbox
                             id={vectorFieldsFieldId + "-text"}
-                            className={styles.settingsSeparator}
                             label={t("labels.vector.options.embedding")}
                             checked={searchTextEmbeddings}
                             onChange={onSearchTextEmbeddingsChange}
@@ -115,7 +114,6 @@ export const VectorSettings = ({
                         />
                         <Checkbox
                             id={vectorFieldsFieldId + "-image"}
-                            className={styles.settingsSeparator}
                             label={t("labels.vector.options.imageEmbedding")}
                             checked={searchImageEmbeddings}
                             onChange={onSearchImageEmbeddingsChange}

app/frontend/src/locales/en/translation.json

@@ -105,11 +105,11 @@
         },
         "useSuggestFollowupQuestions": "Suggest follow-up questions",
         "useAgenticRetrieval": "Use agentic retrieval",
-        "llmInputs": "Inputs for LLM",
+        "llmInputs": "LLM input sources",
         "llmInputsOptions": {
-            "texts": "Texts",
-            "images": "Images",
-            "textAndImages": "Texts and Images"
+            "texts": "Text sources",
+            "images": "Image sources",
+            "textAndImages": "Text and image sources"
         },
         "retrievalMode": {
             "label": "Retrieval mode",
@@ -162,8 +162,8 @@
         "suggestFollowupQuestions": "Asks the LLM to suggest follow-up questions based on the user's query.",
         "textEmbeddings": "When selected, search will use embeddings from the text-only embeddings model of extracted text chunks.",
         "imageEmbeddings": "When selected, search will use embeddings from the multimodal embeddings model of extracted images.",
-        "llmInputs":
-            "Sets what will be sent to the vision model. 'Images and text' sends both images and text to the model, 'Images' sends only images, and 'Text' sends only text.",
+        "llmTextInputs": "When selected, text content from the search results will be sent to the LLM as context.",
+        "llmImageInputs": "When selected, images from the search results will be sent to the LLM as context.",
         "retrievalMode":
             "Sets the retrieval mode for the Azure AI Search query. `Vectors + Text (Hybrid)` uses a combination of vector search and full text search, `Vectors` uses only vector search, and `Text` uses only full text search. Hybrid is generally optimal.",
         "streamChat": "Continuously streams the response to the chat UI as it is generated.",

app/frontend/src/pages/ask/Ask.tsx

@@ -5,7 +5,7 @@ import { Panel, DefaultButton, Spinner } from "@fluentui/react";
 
 import styles from "./Ask.module.css";
 
-import { askApi, configApi, ChatAppResponse, ChatAppRequest, RetrievalMode, LLMInputs, SpeechConfig } from "../../api";
+import { askApi, configApi, ChatAppResponse, ChatAppRequest, RetrievalMode, SpeechConfig } from "../../api";
 import { Answer, AnswerError } from "../../components/Answer";
 import { QuestionInput } from "../../components/QuestionInput";
 import { ExampleList } from "../../components/Example";
@@ -36,7 +36,8 @@ export function Component(): JSX.Element {
     const [useSemanticCaptions, setUseSemanticCaptions] = useState<boolean>(false);
     const [useQueryRewriting, setUseQueryRewriting] = useState<boolean>(false);
     const [reasoningEffort, setReasoningEffort] = useState<string>("");
-    const [llmInputs, setLLMInputs] = useState<LLMInputs>(LLMInputs.Texts);
+    const [sendTextSources, setSendTextSources] = useState<boolean>(true);
+    const [sendImageSources, setSendImageSources] = useState<boolean>(true);
     const [includeCategory, setIncludeCategory] = useState<string>("");
 
     const [excludeCategory, setExcludeCategory] = useState<string>("");
@@ -87,8 +88,8 @@ export function Component(): JSX.Element {
             setShowMultimodalOptions(config.showMultimodalOptions);
             if (config.showMultimodalOptions) {
                 // Set default LLM inputs based on config override or fallback to Texts
-                const defaultLlmInputs = config.ragLlmInputsOverride ? (config.ragLlmInputsOverride as LLMInputs) : LLMInputs.Texts;
-                setLLMInputs(defaultLlmInputs);
+                setSendTextSources(true);
+                setSendImageSources(true);
                 // Set default vector field settings
                 setSearchTextEmbeddings(true);
                 setSearchImageEmbeddings(true);
@@ -162,7 +163,8 @@ export function Component(): JSX.Element {
                         use_groups_security_filter: useGroupsSecurityFilter,
                         search_text_embeddings: searchTextEmbeddings,
                         search_image_embeddings: searchImageEmbeddings,
-                        llm_inputs: llmInputs,
+                        send_text_sources: sendTextSources,
+                        send_image_sources: sendImageSources,
                         language: i18n.language,
                         use_agentic_retrieval: useAgenticRetrieval,
                         ...(seed !== null ? { seed: seed } : {})
@@ -238,7 +240,12 @@ export function Component(): JSX.Element {
                 setUseGroupsSecurityFilter(value);
                 break;
             case "llmInputs":
-                setLLMInputs(value);
+                break;
+            case "sendTextSources":
+                setSendTextSources(value);
+                break;
+            case "sendImageSources":
+                setSendImageSources(value);
                 break;
             case "searchTextEmbeddings":
                 setSearchTextEmbeddings(value);
@@ -375,7 +382,8 @@ export function Component(): JSX.Element {
                     excludeCategory={excludeCategory}
                     includeCategory={includeCategory}
                     retrievalMode={retrievalMode}
-                    llmInputs={llmInputs}
+                    sendTextSources={sendTextSources}
+                    sendImageSources={sendImageSources}
                     searchTextEmbeddings={searchTextEmbeddings}
                     searchImageEmbeddings={searchImageEmbeddings}
                     showSemanticRankerOption={showSemanticRankerOption}

app/frontend/src/pages/chat/Chat.tsx

@@ -7,17 +7,7 @@ import readNDJSONStream from "ndjson-readablestream";
 import appLogo from "../../assets/applogo.svg";
 import styles from "./Chat.module.css";
 
-import {
-    chatApi,
-    configApi,
-    RetrievalMode,
-    ChatAppResponse,
-    ChatAppResponseOrError,
-    ChatAppRequest,
-    ResponseMessage,
-    LLMInputs,
-    SpeechConfig
-} from "../../api";
+import { chatApi, configApi, RetrievalMode, ChatAppResponse, ChatAppResponseOrError, ChatAppRequest, ResponseMessage, SpeechConfig } from "../../api";
 import { Answer, AnswerError, AnswerLoading } from "../../components/Answer";
 import { QuestionInput } from "../../components/QuestionInput";
 import { ExampleList } from "../../components/Example";
@@ -61,7 +51,8 @@ const Chat = () => {
     const [searchImageEmbeddings, setSearchImageEmbeddings] = useState<boolean>(true);
     const [useOidSecurityFilter, setUseOidSecurityFilter] = useState<boolean>(false);
     const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState<boolean>(false);
-    const [llmInputs, setLLMInputs] = useState<LLMInputs>(LLMInputs.TextAndImages);
+    const [sendTextSources, setSendTextSources] = useState<boolean>(true);
+    const [sendImageSources, setSendImageSources] = useState<boolean>(true);
 
     const lastQuestionRef = useRef<string>("");
     const chatMessageStreamEnd = useRef<HTMLDivElement | null>(null);
@@ -109,8 +100,8 @@ const Chat = () => {
             setShowMultimodalOptions(config.showMultimodalOptions);
             if (config.showMultimodalOptions) {
                 // Set default LLM inputs based on config override or fallback to TextAndImages
-                const defaultLlmInputs = config.ragLlmInputsOverride ? (config.ragLlmInputsOverride as LLMInputs) : LLMInputs.TextAndImages;
-                setLLMInputs(defaultLlmInputs);
+                setSendTextSources(true);
+                setSendImageSources(true);
                 // Set default vector fields based on config override or fallback to TextAndImageEmbeddings
                 // Set default vector field settings
                 setSearchTextEmbeddings(true);
@@ -239,7 +230,8 @@ const Chat = () => {
                         use_groups_security_filter: useGroupsSecurityFilter,
                         search_text_embeddings: searchTextEmbeddings,
                         search_image_embeddings: searchImageEmbeddings,
-                        llm_inputs: llmInputs,
+                        send_text_sources: sendTextSources,
+                        send_image_sources: sendImageSources,
                         language: i18n.language,
                         use_agentic_retrieval: useAgenticRetrieval,
                         ...(seed !== null ? { seed: seed } : {})
@@ -357,7 +349,12 @@ const Chat = () => {
                 setUseSuggestFollowupQuestions(value);
                 break;
             case "llmInputs":
-                setLLMInputs(value);
+                break;
+            case "sendTextSources":
+                setSendTextSources(value);
+                break;
+            case "sendImageSources":
+                setSendImageSources(value);
                 break;
             case "searchTextEmbeddings":
                 setSearchTextEmbeddings(value);
@@ -561,7 +558,8 @@ const Chat = () => {
                         includeCategory={includeCategory}
                         retrievalMode={retrievalMode}
                         showMultimodalOptions={showMultimodalOptions}
-                        llmInputs={llmInputs}
+                        sendTextSources={sendTextSources}
+                        sendImageSources={sendImageSources}
                         searchTextEmbeddings={searchTextEmbeddings}
                         searchImageEmbeddings={searchImageEmbeddings}
                         showSemanticRankerOption={showSemanticRankerOption}

todo.txt

@@ -10,8 +10,13 @@ TODO:
       # Inputs for LLM:
       # [X] text sources , use_text_sources = True
       # [X] image sources , use_image_sources = True
-
-
+    Update:
+                // Set default LLM inputs based on config override or fallback to Texts
+                setSendTextSources(true);
+                setSendImageSources(true);
+                // Set default vector field settings
+                setSearchTextEmbeddings(true);
+                setSearchImageEmbeddings(true);
 To decide:
 * For user data lake client, how often should we double check the ACL matches the oid, versus assuming the URLs convey that? (Like when fetching the image?)
    * add a note that we only check owner, not full access control