Use cloud managed platform assistant (text-to-text)

Author: Shellishack

web/components/interface/command-viewer.tsx

@@ -1,5 +1,5 @@
 import useCommands from "@/lib/hooks/use-commands";
-import { Command, MenuAction } from "@/lib/types";
+import { Command } from "@/lib/types";
 import {
   addToast,
   Button,
@@ -25,16 +25,20 @@ const inputPlaceholders = [
 export default function CommandViewer() {
   const editorContext = useContext(EditorContext);
 
-  const { chatWithAssistant } = usePlatformAIAssistant();
+  const { chatWithAssistant, history } = usePlatformAIAssistant();
   const { commands, runCommand, setKeywordFilter } = useCommands();
-  const { registerMenuAction, unregisterMenuAction } = useMenuActions("view");
 
   const [inputPlaceholder, setInputPlaceholder] = useState("");
   const [selectCommandIndex, setSelectCommandIndex] = useState(-1);
   const [inputTextValue, setInputTextValue] = useState("");
+  const [inputAudioValue, setInputAudioValue] = useState<
+    ReadableStream<ArrayBuffer> | undefined
+  >(undefined);
   const [isInputVoice, setIsInputVoice] = useState(false);
   const [isOutputVoice, setIsOutputVoice] = useState(false);
 
+  const historyRef = useRef<HTMLDivElement>(null);
+
   const runCommandCallback = useCallback(
     async (command: Command) => {
       const result = await runCommand(command, {});
@@ -81,6 +85,13 @@ export default function CommandViewer() {
     handlePressedKeys(editorContext?.editorStates.pressedKeys ?? []);
   }, [editorContext?.editorStates.pressedKeys]);
 
+  // Scroll to bottom when history changes
+  useEffect(() => {
+    if (historyRef.current) {
+      historyRef.current.scrollTop = historyRef.current.scrollHeight;
+    }
+  }, [history]);
+
   function handleKeyDown(e: KeyboardEvent) {
     // Prevent default behavior for certain keys
     const key = e.key;
@@ -157,9 +168,12 @@ export default function CommandViewer() {
     } else if (isEnterPressed && !isControlPressed) {
       // Chat with assistant if ctrl is not pressed
       console.log("Chatting with assistant");
-      chatWithAssistant(inputTextValue, isOutputVoice);
-    }
-    else if (isArrowUpPressed) {
+      if (isInputVoice) {
+        chatWithAssistant(inputAudioValue, isOutputVoice);
+      } else {
+        chatWithAssistant(inputTextValue, isOutputVoice);
+      }
+    } else if (isArrowUpPressed) {
       setSelectCommandIndex((prev) =>
         prev === 0 ? commands.length - 1 : prev - 1,
       );
@@ -222,6 +236,29 @@ export default function CommandViewer() {
           onKeyDown={(e) => handleKeyDown(e as any)}
           onKeyUp={(e) => handleKeyUp(e as any)}
         />
+
+        {history.length > 0 && (
+          <div
+            ref={historyRef}
+            className="bg-content1 flex max-h-60 flex-col gap-y-1 overflow-y-auto rounded-2xl p-2 shadow-md"
+          >
+            {history.map((entry, index) => (
+              <div key={index}>
+                {entry.role === "user" ? (
+                  <div className="text-primary-foreground bg-primary rounded-lg p-2 text-sm">
+                    <span className="font-bold">You: </span>
+                    {entry.message.content.text}
+                  </div>
+                ) : (
+                  <div className="text-default-foreground bg-default rounded-lg p-2 text-sm">
+                    <span className="font-bold">Assistant: </span>
+                    {entry.message.content.text}
+                  </div>
+                )}
+              </div>
+            ))}
+          </div>
+        )}
         <div className="bg-content1 rounded-2xl shadow-md">
           <div className="px-3 pt-2">
             <p className="text-sm font-bold whitespace-nowrap">

web/components/interface/editor-toolbar.tsx

@@ -14,7 +14,7 @@ import useRecorder from "@/lib/hooks/use-recorder";
 export default function EditorToolbar() {
   const editorContext = useContext(EditorContext);
 
-  const { chatWithAssistant: runAssistant } = usePlatformAIAssistant();
+  const { chatWithAssistant } = usePlatformAIAssistant();
   const { isRecording, record } = useRecorder();
 
   const [isAgentListModalOpen, setIsAgentListModalOpen] = useState(false);
@@ -110,7 +110,7 @@ export default function EditorToolbar() {
                   onPress={() => {
                     if (!isRecording) {
                       const stream = record();
-                      runAssistant(stream, true);
+                      chatWithAssistant(stream, true);
                     }
                   }}
                   variant={

web/lib/agent/agent-runner.ts

@@ -13,48 +13,69 @@ import toast from "react-hot-toast";
 import { ChatPromptTemplate } from "@langchain/core/prompts";
 import { JsonOutputParser } from "@langchain/core/output_parsers";
 import { fetchAPI } from "../pulse-editor-website/backend";
+import { parseJsonChunk } from "./stream-chunk-parser";
 
 export function getAgentLLMConfig(agent: Agent, methodName: string) {
   const method = agent.availableMethods.find((m) => m.name === methodName);
   return method?.LLMConfig ? method.LLMConfig : agent.LLMConfig;
 }
 
 export async function runAgentMethodCloud(
-  llmConfig: LLMConfig,
   agent: Agent,
   methodName: string,
   args: Record<string, any>,
+  onChunkUpdate?: (chunk: any) => void,
 ): Promise<any> {
   const method = agent.availableMethods.find((m) => m.name === methodName);
   if (!method) {
     throw new Error(`Method ${methodName} not found in agent ${agent.name}.`);
   }
 
-  const prompt = await getPrompt(agent, method, args);
+  const prompt = await getAgentPrompt(agent, method, args);
 
   console.log("Prompt: ", prompt);
 
-  const response = await fetchAPI("/api/inference/platform-assistant", {
-    method: "POST",
-    body: JSON.stringify({
-      prompt: prompt,
-    }),
-  });
+  const response = await fetchAPI(
+    "/api/inference/platform-assistant/text-to-text",
+    {
+      method: "POST",
+      body: JSON.stringify({ prompt }),
+    },
+  );
 
   if (!response.ok) {
     const error = await response.text();
-    throw new Error(error);
+    toast.error(`Error: ${error}`);
+    return;
+  }
+  const stream = response.body;
+  if (!stream) {
+    toast.error("Error: No response from server.");
+    return;
+  }
+  const reader = stream.getReader();
+  const decoder = new TextDecoder("utf-8");
+  let done = false;
+  let result: any;
+
+  while (!done) {
+    const { done: readerDone, value } = await reader.read();
+    done = readerDone;
+    if (value) {
+      const chunk = decoder.decode(value, { stream: !done });
+      const parsedChunks = parseJsonChunk(chunk);
+
+      const latestChunkValue = parsedChunks[parsedChunks.length - 1];
+
+      result = latestChunkValue;
+
+      if (onChunkUpdate) {
+        onChunkUpdate(latestChunkValue);
+      }
+    }
   }
 
-  const llmResult = await response.text();
-
-  console.log("LLM result: ", llmResult);
-
-  const returns = await extractReturns(llmResult);
-
-  console.log("Agent result: ", returns);
-
-  return returns;
+  return result;
 }
 
 export async function runAgentMethodLocal(
@@ -76,7 +97,7 @@ export async function runAgentMethodLocal(
     throw new Error("LLM not found.");
   }
 
-  const prompt = await getPrompt(agent, method, args);
+  const prompt = await getAgentPrompt(agent, method, args);
 
   console.log("Prompt: ", prompt);
 
@@ -112,11 +133,19 @@ function getLLM(llmConfig: LLMConfig, agentName: string, apiKey: string) {
   return llm;
 }
 
-async function getPrompt(
+export async function getAgentPrompt(
   agent: Agent,
-  method: AgentMethod,
+  method: AgentMethod | string,
   args: Record<string, any>,
 ) {
+  if (typeof method === "string") {
+    const foundMethod = agent.availableMethods.find((m) => m.name === method);
+    if (!foundMethod) {
+      throw new Error(`Method ${method} not found in agent ${agent.name}.`);
+    }
+    method = foundMethod;
+  }
+
   const userPromptTemplate = `\
 ${method.prompt}
 

web/lib/agent/built-in-agents/editor-assistant.ts

@@ -223,7 +223,7 @@ Opened views:
 \`\`\`
 Available commands:
 \`\`\`
-{commands}
+{availableCommands}
 \`\`\`
 Project directory tree:
 \`\`\`

web/lib/agent/stream-chunk-parser.ts

@@ -0,0 +1,23 @@
+export function parseJsonChunk(chunk: string) {
+  const jsonObjects = [];
+  // Replace multiple spaces or other delimiters with a single space and trim
+  const cleanedChunk = chunk.trim();
+
+  // Use a regular expression to match JSON objects
+  // This assumes objects are separated by spaces and are valid JSON
+  const jsonRegex = /({[^{}]*})/g;
+  const matches = cleanedChunk.match(jsonRegex);
+
+  if (matches) {
+    for (const match of matches) {
+      try {
+        const parsed = JSON.parse(match);
+        jsonObjects.push(parsed);
+      } catch (error) {
+        console.error("Error parsing JSON object:", match, error);
+      }
+    }
+  }
+
+  return jsonObjects;
+}

web/lib/hooks/use-commands.ts

@@ -16,39 +16,7 @@ import { EditorContext } from "@/components/providers/editor-context-provider";
 import { useTabViewManager } from "./use-tab-view-manager";
 import toast from "react-hot-toast";
 import { v4 } from "uuid";
-
-// Editor built-in commands
-const editorCommands: CommandDefinition[] = [
-  {
-    info: {
-      name: "New Canvas",
-      description: "Create a new blank canvas",
-      parameters: {},
-    },
-    handler: async (args: any) => {
-      // Implementation of creating a new canvas
-      console.log("Creating a new canvas with args:", args);
-      return { success: true };
-    },
-  },
-  {
-    info: {
-      name: "Open File",
-      description: "Open a file in a new tab",
-      parameters: {
-        filePath: {
-          type: "string",
-          description: "Path to the file to open",
-        },
-      },
-    },
-    handler: async (args: any) => {
-      // Implementation of opening a file
-      console.log("Opening file with args:", args);
-      return { success: true };
-    },
-  },
-];
+import { useMenuActions } from "./use-menu-actions";
 
 /**
  *  Use commands in active tab.
@@ -65,11 +33,45 @@ export default function useCommands() {
     createAppViewInCanvasView,
     findAppInTabView,
   } = useTabViewManager();
+  const { menuActions } = useMenuActions();
 
   const [commands, setCommands] = useState<Command[]>([]);
 
   const [keyword, setKeyword] = useState<string | undefined>(undefined);
 
+  // Editor built-in commands
+  const editorCommands: CommandDefinition[] = [
+    {
+      info: {
+        name: "New Workflow",
+        description: "Create a new blank Workflow",
+        parameters: {},
+      },
+      handler: async (args: any) => {
+        // Implementation of creating a new workflow
+        menuActions?.find((a) => a.name === "New Workflow")?.actionFunc();
+        return { success: true };
+      }
+    },
+    {
+      info: {
+        name: "Open File",
+        description: "Open a file in a new tab",
+        parameters: {
+          filePath: {
+            type: "string",
+            description: "Path to the file to open",
+          },
+        },
+      },
+      handler: async (args: any) => {
+        // Implementation of opening a file
+        console.log("Opening file with args:", args);
+        return { success: true };
+      },
+    },
+  ];
+
   // Update editor commands
   useEffect(() => {
     const newEditorCommands = editorCommands.map((cmd) => ({
@@ -235,7 +237,6 @@ export default function useCommands() {
     setKeyword(newKeyword);
   }
 
-
   return {
     runCommand,
     commands,