Change action result to display as append block (#1425)

- Action result display content should be appended as block so that it
can work with using `actionIO` within the action. Other display option
can be access thru `actionIO`
- Make sure the chat `generateResponse` and `lookupAndAnswer` doesn't
display twice.
- Rename the action result's `literalText` to `historyText` to indicate
that the text is used for chat history.
- Change `lookupAndAnswer` to return the resulting text in `historyText`
as well along with the entities for used for chat history.

Author: curtisman

ts/packages/agentSdk/src/action.ts

@@ -21,8 +21,8 @@ export type ActionResultActivityContext = Omit<
 > | null;
 
 export type ActionResultSuccess = {
-    literalText?: string | undefined;
-    displayContent: DisplayContent;
+    historyText?: string | undefined;
+    displayContent: DisplayContent; // the display content to be appended with "block" mode
     entities: Entity[];
     resultEntity?: Entity | undefined;
     dynamicDisplayId?: string | undefined;

ts/packages/agentSdk/src/helpers/actionHelpers.ts

@@ -11,17 +11,17 @@ import { DisplayMessageKind } from "../display.js";
 import { Entity } from "../memory.js";
 
 export function createActionResultNoDisplay(
-    literalText: string,
+    historyText: string,
     entities?: Entity[] | undefined,
 ): ActionResultSuccessNoDisplay {
     return {
-        literalText,
+        historyText,
         entities: entities ? entities : [],
     };
 }
 
 export function createActionResult(
-    literalText: string,
+    displayAndHistoryText: string,
     options?:
         | {
               kind?: DisplayMessageKind;
@@ -39,7 +39,7 @@ export function createActionResult(
               : options;
 
     return {
-        literalText,
+        historyText: displayAndHistoryText,
         entities: entities
             ? Array.isArray(entities)
                 ? entities
@@ -48,31 +48,31 @@ export function createActionResult(
         displayContent: displayOptions
             ? {
                   type: "text",
-                  content: literalText,
+                  content: displayAndHistoryText,
                   ...displayOptions,
               }
-            : literalText,
+            : displayAndHistoryText,
     };
 }
 
 export function createActionResultFromTextDisplay(
     displayText: string,
-    literalText?: string,
+    historyText?: string,
 ): ActionResultSuccess {
     return {
-        literalText,
+        historyText,
         entities: [],
         displayContent: displayText,
     };
 }
 
 export function createActionResultFromHtmlDisplay(
     displayText: string,
-    literalText?: string,
+    historyText?: string,
     entities?: Entity[] | undefined,
 ): ActionResultSuccess {
     return {
-        literalText,
+        historyText,
         entities: entities ? entities : [],
         displayContent: {
             type: "html",
@@ -83,10 +83,10 @@ export function createActionResultFromHtmlDisplay(
 
 export function createActionResultFromHtmlDisplayWithScript(
     displayText: string,
-    literalText?: string,
+    historyText?: string,
 ): ActionResultSuccess {
     return {
-        literalText,
+        historyText,
         entities: [],
         displayContent: {
             type: "iframe",
@@ -104,13 +104,13 @@ export function createActionResultFromHtmlDisplayWithScript(
  */
 export function createActionResultFromMarkdownDisplay(
     markdownText: string | string[],
-    literalText?: string,
+    historyText?: string,
     entities: Entity[] = [],
     resultEntity?: Entity,
 ): ActionResultSuccess {
     return {
-        literalText:
-            literalText ??
+        historyText:
+            historyText ??
             (Array.isArray(markdownText)
                 ? markdownText.join("\n")
                 : markdownText),

ts/packages/agents/browser/src/agent/actionHandler.mts

@@ -1779,7 +1779,7 @@ async function handleWebsiteAction(
             );
             return {
                 success: !statsResult.error,
-                result: statsResult.literalText || "Stats retrieved",
+                result: statsResult.historyText || "Stats retrieved",
                 error: statsResult.error,
             };
 
@@ -1976,8 +1976,8 @@ async function handleWebsiteLibraryStats(
 
         // Parse and format the response - extract text from ActionResult
         let responseText = "";
-        if (statsResult.literalText) {
-            responseText = statsResult.literalText;
+        if (statsResult.historyText) {
+            responseText = statsResult.historyText;
         } else if (statsResult.displayContent) {
             // Handle different types of display content
             if (typeof statsResult.displayContent === "string") {

ts/packages/agents/chat/src/chatResponseHandler.ts

@@ -10,7 +10,6 @@ import {
 
 import { ActionContext, AppAgent, TypeAgentAction } from "@typeagent/agent-sdk";
 import {
-    createActionResult,
     createActionResultFromHtmlDisplay,
     createActionResultNoDisplay,
 } from "@typeagent/agent-sdk/helpers/action";
@@ -63,8 +62,7 @@ async function handleChatResponse(
     console.log(JSON.stringify(chatAction, undefined, 2));
     switch (chatAction.actionName) {
         case "generateResponse": {
-            return generateReponse(chatAction, context);
-            break;
+            return generateResponse(chatAction, context);
         }
 
         case "showImageFile":
@@ -79,7 +77,7 @@ async function handleChatResponse(
     }
 }
 
-async function generateReponse(
+async function generateResponse(
     generateResponseAction: GenerateResponseAction,
     context: ActionContext,
 ) {
@@ -92,23 +90,29 @@ async function generateReponse(
             "Got generated text: " + generatedText.substring(0, 100) + "...",
         );
 
-        const needDisplay =
-            context.streamingContext !== generatedText ||
-            generateResponseAction.parameters.relatedFiles;
-        let result;
-        if (needDisplay) {
-            if (generateResponseAction.parameters.relatedFiles) {
-                result = createActionResultFromHtmlDisplay(
-                    `<div>${generatedText}</div><div class='chat-smallImage'>${await rehydrateImages(context, generateResponseAction.parameters.relatedFiles!)}</div>`,
-                );
-            } else {
-                result = createActionResult(generatedText, true);
-            }
-        } else {
-            result = createActionResultNoDisplay(generatedText);
+        const streamingContext = context.streamingContext;
+        context.streamingContext = undefined; // clear the streaming context
+        if (streamingContext !== generatedText) {
+            // Either we didn't stream, or we streamed a different text.
+            // REVIEW: what happens to the speaking text that was streamed?
+            context.actionIO.setDisplay({
+                type: "text",
+                content: generatedText,
+                speak: streamingContext === undefined,
+            });
         }
 
-        let entities = parameters.generatedTextEntities || [];
+        // Add the related files.
+        if (generateResponseAction.parameters.relatedFiles) {
+            context.actionIO.appendDisplay(
+                `<div class='chat-smallImage'>${await rehydrateImages(context, generateResponseAction.parameters.relatedFiles!)}</div>`,
+                "block",
+            );
+        }
+
+        const result = createActionResultNoDisplay(generatedText);
+
+        const entities = parameters.generatedTextEntities || [];
         if (parameters.userRequestEntities !== undefined) {
             result.entities = parameters.userRequestEntities.concat(entities);
         }

ts/packages/agents/greeting/src/greetingCommandHandler.ts

@@ -19,7 +19,6 @@ import { ChatModelWithStreaming, CompletionSettings, openai } from "aiclient";
 import { PromptSection, Result } from "typechat";
 import {
     displayError,
-    displayResult,
     displayStatus,
 } from "@typeagent/agent-sdk/helpers/display";
 import {
@@ -153,7 +152,10 @@ export class GreetingCommandHandler implements CommandHandlerNoParams {
                         context,
                     )) as ActionResultSuccess;
 
-                    displayResult(result.literalText!, context);
+                    context.actionIO.appendDisplay(
+                        result.displayContent,
+                        "block",
+                    );
                     break;
 
                 // case "contextualGreetingAction":

ts/packages/agents/player/src/agent/playerHandlers.ts

@@ -280,7 +280,7 @@ async function getPlayerDynamicDisplay(
         const status = await htmlStatus(context.agentContext.spotify);
         return {
             content:
-                type === "html" ? status.displayContent : status.literalText!,
+                type === "html" ? status.displayContent : status.historyText!,
 
             nextRefreshMs: 1000,
         };

ts/packages/agents/player/src/client.ts

@@ -223,7 +223,7 @@ async function htmlTrackNames(
 
     const actionResult: ActionResult = {
         displayContent,
-        literalText: "",
+        historyText: "",
         entities: [],
     };
     let prevUrl = "";
@@ -279,7 +279,7 @@ async function htmlTrackNames(
             }
         }
         displayContent.content += "</ol></div>";
-        actionResult.literalText =
+        actionResult.historyText =
             "Updated the current track list with the numbered list of tracks on the screen";
     } else if (selectedTracks.length === 1) {
         const track = selectedTracks[0];
@@ -312,7 +312,7 @@ async function htmlTrackNames(
         const litArtists =
             litArtistsPrefix +
             track.artists.map((artist) => artist.name).join(", ");
-        actionResult.literalText = `Now playing: ${track.name} from album ${track.album.name} with ${litArtists}`;
+        actionResult.historyText = `Now playing: ${track.name} from album ${track.album.name} with ${litArtists}`;
         if (track.album.images.length > 0 && track.album.images[0].url) {
             displayContent.content = "<div class='track-list scroll_enabled'>";
             displayContent.content +=

ts/packages/agents/player/src/playback.ts

@@ -73,7 +73,7 @@ function htmlPlaybackStatus(
             }
             const pp = status.is_playing ? "" : "(paused)";
             const album = status.item.album.name;
-            actionResult.literalText += `Now playing${pp}: ${status.item.name} from album ${album} with ${artists}`;
+            actionResult.historyText += `Now playing${pp}: ${status.item.name} from album ${album} with ${artists}`;
             actionResult.entities.push({
                 name: status.item.name,
                 type: ["track"],
@@ -107,7 +107,7 @@ export async function htmlStatus(context: IClientContext) {
         content: "<div data-group='status'>Status...",
     };
     const actionResult: ActionResultSuccess = {
-        literalText: "",
+        historyText: "",
         entities: [],
         displayContent,
     };
@@ -117,10 +117,10 @@ export async function htmlStatus(context: IClientContext) {
         const aux = `Volume is ${activeDevice.volume_percent}%. ${status.shuffle_state ? "Shuffle on" : ""}`;
         displayContent.content += `<div>Active device: ${activeDevice.name} of type ${activeDevice.type}</div>`;
         displayContent.content += `<div>${aux}</div>`;
-        actionResult.literalText += `\nActive device: ${activeDevice.name} of type ${activeDevice.type}\n${aux}`;
+        actionResult.historyText += `\nActive device: ${activeDevice.name} of type ${activeDevice.type}\n${aux}`;
     } else {
         displayContent.content += "<div>Nothing playing.</div>";
-        actionResult.literalText = "Nothing playing.";
+        actionResult.historyText = "Nothing playing.";
     }
     displayContent.content += "</div>";
     actionResult.dynamicDisplayId = "status";

ts/packages/dispatcher/src/context/dispatcher/dispatcherAgent.ts

@@ -182,7 +182,7 @@ async function clarifyWithLookup(
 
     if (
         lookupResult.error !== undefined ||
-        lookupResult.literalText === undefined
+        lookupResult.historyText === undefined
     ) {
         return undefined;
     }
@@ -196,7 +196,7 @@ async function clarifyWithLookup(
 
     history.promptSections.push({
         role: "assistant",
-        content: lookupResult.literalText,
+        content: lookupResult.historyText,
     });
 
     const translationResult = await translateRequest(

ts/packages/dispatcher/src/context/memory.ts

@@ -201,8 +201,8 @@ export function addActionResultToMemory(
 
         addResultToMemory(
             context,
-            result.literalText
-                ? result.literalText
+            result.historyText
+                ? result.historyText
                 : `Action ${getFullActionName(executableAction)} completed.`,
             schemaName,
             combinedEntities,
@@ -227,19 +227,19 @@ export async function lookupAndAnswerFromMemory(
         throw new Error(`Conversation memory search failed: ${result.message}`);
     }
 
-    const literalText: string[] = [];
+    const historyText: string[] = [];
     for (const [searchResult, answer] of result.data) {
         debug("Conversation memory search result:", searchResult);
         if (answer.type === "Answered") {
-            literalText.push(answer.answer!);
+            historyText.push(answer.answer!);
             displayResult(answer.answer!, context);
         } else {
-            literalText.push(answer.whyNoAnswer!);
+            historyText.push(answer.whyNoAnswer!);
             displayError(answer.whyNoAnswer!, context);
         }
     }
     // TODO: how about entities?
-    return literalText;
+    return historyText;
 }
 
 function ensureMemory(context: ActionContext<CommandHandlerContext>) {