Merge pull request #55 from chrisglein/imageFix

Restore ability to opt-out of image responses

Author: chrisglein

README.md

@@ -36,7 +36,8 @@ There is a feedback dialog on each AI response. This is not yet implemented.
 
 <img width="255" alt="image" src="https://user-images.githubusercontent.com/26607885/223870036-e8100f80-3360-4114-abb7-52e284039063.png">
 
-As is the "Regenerate response" button.
+### Regenerate
+Pressing this button will result in a new OpenAI query for the previous response.
 
 <img width="138" alt="image" src="https://user-images.githubusercontent.com/26607885/223875964-0a46490d-7d9a-4fe3-b57b-46ad7042e57d.png">
 
@@ -61,9 +62,9 @@ The app's settings are handled by a `SettingsContext` object, which has dialog U
 | File | Type | Information |
 | --- | --- | --- |
 | App.tsx | `App` | Root of the app, publishes the `StylesContext` and `SettingsContext` |
-| AppContent.tsx | `ChatSession` | Owns the `ChatElement` list, and handles any writes to that list |
-| AppContent.tsx | `AutomatedChatSession` | Populates the `ChatSession` with either scripted responses or by creating components that query OpenAi |
-| Chat.tsx | `Chat` | The scrolling list of chat entries. Publishes the `FeedbackContext`, `ChatHistoryContext`, and `ChatScrollContext` services. Hosts a `ChatEntry` for the user input. Hosts the dialogs of the app (`FeedbackPopup` and `SettingsPopups`).
+| ChatSession.tsx | `ChatSession` | Owns the `ChatElement` list, publishes the `ChatHistoryContext`, and handles any writes to that list |
+| ChatSession.tsx | `AutomatedChatSession` | Populates the `ChatSession` with either scripted responses or by creating components that query OpenAi |
+| Chat.tsx | `Chat` | The scrolling list of chat entries. Publishes the `FeedbackContext`, and `ChatScrollContext` services. Hosts a `ChatEntry` for the user input. Hosts the dialogs of the app (`FeedbackPopup` and `SettingsPopups`).
 | Chat.tsx | `ChatEntry` | Takes in the user's text input | 
 | Feedback.tsx | `FeedbackPopup` | Popup for giving feedback on AI responses | 
 | Settings.tsx | `SettingsPopup` | Popup that shows controls for modifying the `SettingsContext` | 

src/AiQuery.tsx

@@ -4,11 +4,7 @@ import {
   OpenAiApi,
   CallOpenAi,
 } from './OpenAI';
-import {
-  AiSection,
-  AiImageResponse,
-  AiTextResponse,
-} from './AiResponse';
+import { AiSection } from './AiResponse';
 import { 
   ChatSource,
   ChatContent,
@@ -20,34 +16,36 @@ import { SettingsContext } from './Settings';
 // Component that drives the queries to OpenAi to respond to a prompt
 type AiSectionWithQueryProps = {
   prompt: string;
+  intent?: string;
   id: number;
-  onResponse: ({prompt, response, contentType} : { prompt: string, response: string, contentType: ChatContent} ) => void;
+  onResponse: ({prompt, intent, response, contentType} : { prompt: string, intent?: string, response: string, contentType: ChatContent} ) => void;
 };
-function AiSectionWithQuery({prompt, id, onResponse}: AiSectionWithQueryProps): JSX.Element {
+function AiSectionWithQuery({prompt, intent, id, onResponse}: AiSectionWithQueryProps): JSX.Element {
   const settingsContext = React.useContext(SettingsContext);
   const chatScroll = React.useContext(ChatScrollContext);
   const chatHistory = React.useContext(ChatHistoryContext);
   const [isLoading, setIsLoading] = React.useState(true);
-  const [queryResult, setQueryResult] = React.useState<string | undefined>(undefined);
-  const [error, setError] = React.useState<string | undefined>(undefined);
   const [isRequestForImage, setIsRequestForImage] = React.useState<boolean | undefined>(undefined);
   const [imagePrompt, setImagePrompt] = React.useState<string | undefined>(undefined);
 
   // First determine the intent of the prompt
   const imageIntentSentinel = "[IMAGE]";
   React.useEffect(() => {
     setIsLoading(true);
-    setError(undefined);
     setIsRequestForImage(undefined);
     setImagePrompt(undefined);
+    if (intent === 'text') {
+      setIsRequestForImage(false);
+      return;
+    }
     CallOpenAi({
       api: OpenAiApi.Completion,
       apiKey: settingsContext.apiKey,
       instructions: `You are an intuitive assistant helping the user with a project. Your only job is need to determine the primary intent of the user's last prompt.
 If and only if you are absolutely certain the user's primary intent is to see an image, respond with exactly the string "${imageIntentSentinel}". Otherwise, respond with your description of their intent.`,
       identifier: "INTENT:",
       prompt: prompt,
-      onError: (error) => {
+      onError: () => {
         setIsRequestForImage(false);
       },
       onResult: (result) => {
@@ -83,7 +81,7 @@ Where items enclosed in brackets ([]) would be replaced with an appropriate sugg
 Respond with the image prompt string in the required format. Do not respond conversationally.`,
         identifier: "KEYWORDS:",
         prompt: prompt,
-        onError: (error) => {
+        onError: () => {
           setIsRequestForImage(false);
         },
         onResult: (result) => {
@@ -107,14 +105,12 @@ Respond with the image prompt string in the required format. Do not respond conv
           filter((entry) => { return entry.text !== undefined && entry.id < id; }).
           map((entry) => { return {role: entry.type == ChatSource.Human ? "user" : "assistant", "content": entry.text ?? ""} }),
         onError: (error) => {
-          setError(error);
           onResponse({
             prompt: prompt,
             response: error ?? "",
             contentType: ChatContent.Error});
         },
         onResult: (result) => {
-          setQueryResult(result);
           onResponse({
             prompt: prompt,
             response: result ?? "", 
@@ -133,14 +129,12 @@ Respond with the image prompt string in the required format. Do not respond conv
           identifier: "IMAGE-ANSWER:",
           prompt: imagePrompt,
           onError: (error) => {
-            setError(error);
             onResponse({
               prompt: imagePrompt,
               response: error ?? "",
               contentType: ChatContent.Error});
           },
           onResult: (result) => {
-            setQueryResult(result);
             onResponse({
               prompt: imagePrompt,
               response: result ?? "",
@@ -157,20 +151,16 @@ Respond with the image prompt string in the required format. Do not respond conv
   return (
     <AiSection isLoading={isLoading}>
       {
-        // TODO: All of this can go away now, once images are working in new model
-        (isLoading || error) ?
-          <Text style={{color: 'crimson'}}>{error}</Text>
-        : isRequestForImage ? 
-          <AiImageResponse
-            imageUrl={queryResult}
-            prompt={imagePrompt}
-            rejectImage={() => {
-              setIsRequestForImage(false);
-              setQueryResult(undefined);
-          }}/>
-        : // Not an error, not an image
-          <AiTextResponse
-            text={queryResult}/>
+        (isLoading ?
+          isRequestForImage === undefined ?
+            <Text>Identifying intent...</Text> :
+            isRequestForImage === true ?
+              imagePrompt === undefined ?
+                <Text>Generating keywords for an image...</Text> :
+                <Text>Generating image...</Text> :
+              <Text>Generating text...</Text>
+            :
+          <Text>Done loading</Text>)
       }
     </AiSection>
   )

src/AiResponse.tsx

@@ -14,7 +14,8 @@ import {
 } from './Controls';
 import {
   ChatElement,
-  ChatContent
+  ChatContent,
+  ChatHistoryContext
 } from './Chat';
 import { StylesContext } from './Styles';
 import { FeedbackContext } from './Feedback';
@@ -38,7 +39,7 @@ function AiImageResponse({imageUrl, prompt, rejectImage}: AiImageResponseProps):
         <View style={{alignSelf: 'flex-end', alignItems: 'flex-end'}}>
           <Button
             title="I didn't want to see an image"
-            onPress={() => {rejectImage()}}/>
+            onPress={() => rejectImage()}/>
           <Button
             title="Show me more"
             onPress={() => console.log("Not yet implemented")}/>
@@ -143,9 +144,11 @@ function AiSection({children, isLoading, contentShownOnHover}: AiSectionProps):
 }
 
 type AiSectionContentProps = {
+  id: number,
   content: ChatElement;
 }
-function AiSectionContent({content}: AiSectionContentProps): JSX.Element {
+function AiSectionContent({id, content}: AiSectionContentProps): JSX.Element {
+  const chatHistory = React.useContext(ChatHistoryContext);
   return (
     <AiSection>
       {(() => {
@@ -156,7 +159,7 @@ function AiSectionContent({content}: AiSectionContentProps): JSX.Element {
             return <AiImageResponse
               imageUrl={content.text}
               prompt={content.prompt}
-              rejectImage={() => console.log("Not yet implemented")}/>; // TODO: This would need to reset back to the text prompt
+              rejectImage={() => chatHistory.modifyResponse(id, {intent: 'text', text: undefined})}/>;
           default:
           case ChatContent.Text:
             return <AiTextResponse text={content.text}/>

src/App.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import { Appearance } from 'react-native';
-import { ChatSession } from './AppContent';
+import { ChatSession } from './ChatSession';
 import {
   StylesContext,
   CreateStyles,

src/Chat.tsx

@@ -2,7 +2,6 @@ import React from 'react';
 import {
   Button,
   ScrollView,
-  Text,
   TextInput,
   View,
 } from 'react-native';
@@ -30,6 +29,7 @@ type ChatElement = {
   id: number;
   type: ChatSource;
   contentType: ChatContent;
+  intent?: string;
   prompt?: string;
   text?: string;
   content?: JSX.Element;
@@ -38,7 +38,11 @@ type ChatElement = {
 // Context for read-only access to the chat log
 const ChatHistoryContext = React.createContext<{
   entries: ChatElement[];
-}>({entries: []});
+  modifyResponse: (id: number, delta?: any) => void;
+}>({
+  entries: [],
+  modifyResponse: () => {},
+});
 
 // Context for being able to drive the chat scroller
 const ChatScrollContext = React.createContext<{
@@ -93,12 +97,11 @@ type ChatProps = {
   entries: ChatElement[];
   humanText? : string;
   onPrompt: (prompt: string) => void;
-  onResponse: ({prompt, response, contentType, entryId} : { prompt: string, response: string, contentType: ChatContent, entryId: number} ) => void;
-  regenerateResponse: () => void;
   clearConversation: () => void;
 };
-function Chat({entries, humanText, onPrompt, onResponse, regenerateResponse, clearConversation}: ChatProps): JSX.Element {
+function Chat({entries, humanText, onPrompt, clearConversation}: ChatProps): JSX.Element {
   const styles = React.useContext(StylesContext);
+  const chatHistory = React.useContext(ChatHistoryContext);
   const [showFeedbackPopup, setShowFeedbackPopup] = React.useState(false);
   const [showSettingsPopup, setShowSettingsPopup] = React.useState(false);
   const [feedbackIsPositive, setFeedbackIsPositive] = React.useState(false);
@@ -120,70 +123,77 @@ function Chat({entries, humanText, onPrompt, onResponse, regenerateResponse, cle
 
   return (
     <FeedbackContext.Provider value={feedbackContext}>
-      <ChatHistoryContext.Provider value={{entries: entries}}>
-        <ChatScrollContext.Provider value={{scrollToEnd: scrollToEnd}}>
-          <View style={styles.appContent}>
-            <ScrollView
-              contentInsetAdjustmentBehavior="automatic"
-              ref={scrollViewRef}
-              style={{flexShrink: 1}}>
-              <View
-                style={{gap: 12}}>
-                {// For each item in the chat log, render the appropriate component
-                entries.map((entry) => (
-                  <View key={entry.id}>
-                    {
-                      entry.type === ChatSource.Human ? 
-                        // Human inputs are always plain text
-                        <HumanSection content={entry.text}/> :
-                        entry.content ?
-                          // The element may have provided its own UI
-                          entry.content :
-                          // Otherwise, either render the completed query or start a query to get the resolved text
-                          entry.text ?
-                            <AiSectionContent content={entry}/> : 
-                            <AiSectionWithQuery
-                              id={entry.id}
-                              prompt={entry.prompt ?? ""}
-                              onResponse={({prompt, response, contentType}) => onResponse({prompt: prompt, response: response, contentType: contentType, entryId: entry.id})}/>
-                    }
-                  </View>
-                ))}
-                {(entries.length > 0) &&
-                  <View style={{alignSelf: 'center'}}>
-                    <Button title="🔁 Regenerate response" onPress={() => regenerateResponse()}/>
-                  </View>
-                }
-              </View>
-            </ScrollView>
+      <ChatScrollContext.Provider value={{scrollToEnd: scrollToEnd}}>
+        <View style={styles.appContent}>
+          <ScrollView
+            contentInsetAdjustmentBehavior="automatic"
+            ref={scrollViewRef}
+            style={{flexShrink: 1}}>
             <View
-              style={{flexShrink: 0, marginBottom: 12}}>
-              <HumanSection
-                disableEdit={true}
-                disableCopy={true}
-                contentShownOnHover={
-                  <HoverButton content="⚙️" onPress={() => setShowSettingsPopup(true)}/>
-                }>
-                <ChatEntry
-                  defaultText={humanText}
-                  submit={(newEntry) => {
-                    onPrompt(newEntry);
-                    scrollToEnd();
-                  }}
-                  clearConversation={clearConversation}/>
-              </HumanSection>
+              style={{gap: 12}}>
+              {// For each item in the chat log, render the appropriate component
+              entries.map((entry) => (
+                <View key={entry.id}>
+                  {
+                    entry.type === ChatSource.Human ? 
+                      // Human inputs are always plain text
+                      <HumanSection content={entry.text}/> :
+                      entry.content ?
+                        // The element may have provided its own UI
+                        entry.content :
+                        // Otherwise, either render the completed query or start a query to get the resolved text
+                        entry.text ?
+                          <AiSectionContent
+                            id={entry.id}
+                            content={entry}/> : 
+                          <AiSectionWithQuery
+                            id={entry.id}
+                            prompt={entry.prompt ?? ""}
+                            intent={entry.intent}
+                            onResponse={({prompt, response, contentType}) => 
+                              chatHistory.modifyResponse(entry.id, {prompt: prompt, text: response, contentType: contentType})}/>
+                  }
+                </View>
+              ))}
+              {(entries.length > 0) &&
+                <View style={{alignSelf: 'center'}}>
+                  <Button
+                    title="🔁 Regenerate response"
+                    onPress={() => {
+                      // Clear the response for the last entry
+                      chatHistory.modifyResponse(entries.length - 1, {text: undefined});
+                    }}/>
+                </View>
+              }
             </View>
-            { (showFeedbackPopup || showSettingsPopup) && <View style={styles.popupBackground}/> }
-            <FeedbackPopup
-              show={showFeedbackPopup}
-              isPositive={feedbackIsPositive}
-              close={() => setShowFeedbackPopup(false)}/>
-            <SettingsPopup
-              show={showSettingsPopup}
-              close={() => setShowSettingsPopup(false)}/>
+          </ScrollView>
+          <View
+            style={{flexShrink: 0, marginBottom: 12}}>
+            <HumanSection
+              disableEdit={true}
+              disableCopy={true}
+              contentShownOnHover={
+                <HoverButton content="⚙️" onPress={() => setShowSettingsPopup(true)}/>
+              }>
+              <ChatEntry
+                defaultText={humanText}
+                submit={(newEntry) => {
+                  onPrompt(newEntry);
+                  scrollToEnd();
+                }}
+                clearConversation={clearConversation}/>
+            </HumanSection>
           </View>
-        </ChatScrollContext.Provider>
-      </ChatHistoryContext.Provider>
+          { (showFeedbackPopup || showSettingsPopup) && <View style={styles.popupBackground}/> }
+          <FeedbackPopup
+            show={showFeedbackPopup}
+            isPositive={feedbackIsPositive}
+            close={() => setShowFeedbackPopup(false)}/>
+          <SettingsPopup
+            show={showSettingsPopup}
+            close={() => setShowSettingsPopup(false)}/>
+        </View>
+      </ChatScrollContext.Provider>
     </FeedbackContext.Provider>
   );
 }