Change voice input loading/unloading time and update new models

Author: Shellishack

mobile/android/build.gradle

@@ -7,7 +7,7 @@ buildscript {
         mavenCentral()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:8.2.1'
+        classpath 'com.android.tools.build:gradle:8.7.3'
         classpath 'com.google.gms:google-services:4.4.0'
 
         // NOTE: Do not place your application dependencies here; they belong

mobile/android/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

web/app/page.tsx

@@ -1,134 +1,20 @@
 "use client";
 
-import { useMicVAD, utils } from "@/lib/hooks/use-mic-vad";
-import { BaseLLM } from "@/lib/llm/llm";
-import { BaseSTT } from "@/lib/stt/stt";
-import { useContext, useEffect, useRef } from "react";
-import toast from "react-hot-toast";
-import { CodeEditorAgent } from "@/lib/agents/code-editor-agent";
-import { BaseTTS } from "@/lib/tts/tts";
 import EditorToolbar from "@/components/interface/editor-toolbar";
 import { EditorContext } from "@/components/providers/editor-context-provider";
+import Voice from "@/components/tools/voice";
 import ViewDisplayArea from "@/components/views/file-view-display-area";
-import { useViewManager } from "@/lib/hooks/use-view-manager";
+import { useContext } from "react";
 
 export default function Home() {
   const editorContext = useContext(EditorContext);
-  const { updateFileView, activeFileView } = useViewManager();
-
-  // TODO: Use a timer to stop recorder if no speech is detected for more than 30 seconds
-  const isProcessingRef = useRef(false);
-  const vad = useMicVAD({
-    startOnLoad: false,
-    baseAssetPath: "/vad/",
-    onnxWASMBasePath: "/vad/",
-    positiveSpeechThreshold: 0.75,
-    onSpeechStart: () => {
-      if (!isProcessingRef.current) {
-        const sttModel = editorContext?.aiModelConfig.getSTTModel();
-        const llmModel = editorContext?.aiModelConfig.getLLMModel();
-        const ttsModel = editorContext?.aiModelConfig.getTTSModel();
-        let isConfigured = true;
-        if (!sttModel) {
-          toast.error("STT model is not configured in settings.");
-          isConfigured = false;
-        }
-        if (!llmModel) {
-          toast.error("LLM model is not configured in settings.");
-          isConfigured = false;
-        }
-        if (!ttsModel) {
-          toast.error("TTS model is not configured in settings.");
-          isConfigured = false;
-        }
-
-        if (!isConfigured) {
-          return;
-        }
-
-        editorContext?.setEditorStates((prev) => ({
-          ...prev,
-          isListening: true,
-        }));
-      }
-    },
-    onSpeechEnd: (audio) => {
-      if (!isProcessingRef.current) {
-        isProcessingRef.current = true;
-        const wavBuffer = utils.encodeWAV(audio);
-        const blob = new Blob([wavBuffer], { type: "audio/wav" });
-        console.log("Speech end\n", blob);
-
-        const sttModel = editorContext?.aiModelConfig.getSTTModel() as BaseSTT;
-        const llmModel = editorContext?.aiModelConfig.getLLMModel() as BaseLLM;
-        const ttsModel = editorContext?.aiModelConfig.getTTSModel() as BaseTTS;
-
-        const agent = new CodeEditorAgent(sttModel, llmModel, ttsModel);
-        editorContext?.setEditorStates((prev) => ({
-          ...prev,
-          isListening: false,
-          isThinking: true,
-        }));
-        agent
-          .generateAgentCompletion(
-            activeFileView?.fileContent || "",
-            activeFileView?.selections || [],
-            {
-              audio: blob,
-            },
-          )
-          .then((result) => {
-            const changes = agent.getLineChanges(result.text.codeCompletion);
-            editorContext?.setEditorStates((prev) => ({
-              ...prev,
-              isThinking: false,
-            }));
-
-            // Apply changes
-            updateFileView({
-              filePath: activeFileView?.filePath || "",
-              // suggestedLines: changes,
-              fileContent: result.text.codeCompletion,
-              isActive: true,
-            });
-
-            // Play the audio in the blob
-            if (result.audio) {
-              const audio = new Audio(URL.createObjectURL(result.audio));
-              audio.onended = () => {
-                console.log("Audio ended");
-                editorContext?.setEditorStates((prev) => ({
-                  ...prev,
-                  isSpeaking: false,
-                }));
-                isProcessingRef.current = false;
-              };
-              editorContext?.setEditorStates((prev) => ({
-                ...prev,
-                isSpeaking: true,
-              }));
-              audio.play();
-              return;
-            }
-            isProcessingRef.current = false;
-          });
-      }
-    },
-  });
-
-  // Toggle recording
-  useEffect(() => {
-    if (editorContext?.editorStates?.isRecording) {
-      vad.start();
-    } else {
-      vad.stop();
-    }
-  }, [editorContext?.editorStates, vad]);
 
   return (
     <div className="flex h-full w-full flex-col">
       <EditorToolbar />
       <ViewDisplayArea />
+
+      {editorContext?.editorStates.isRecording && <Voice />}
     </div>
   );
 }

web/components/misc/icon.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { useTheme } from "next-themes";
+import Image from "next/image";
 
 export default function Icon({
   name,
@@ -24,33 +25,55 @@ export default function Icon({
   }
   if (name && uri) {
     throw new Error(
-      "Icon component requires either a name or a uri prop, not both."
+      "Icon component requires either a name or a uri prop, not both.",
     );
   }
 
   if (name) {
     return (
-      <div className="flex justify-center items-center w-full h-full">
-        <span
-          className={
-            `material-icons${variant ? "-" + variant : ""}` +
-            (className ? " " + className : "")
-          }
-        >
-          {name}
-        </span>
-      </div>
+      <span
+        className={
+          `material-icons${variant ? "-" + variant : ""}` +
+          (className ? " " + className : "")
+        }
+      >
+        {name}
+      </span>
     );
   }
 
   if (!isThemed) {
     const iconUri = uri + extension;
-    return <img src={iconUri} alt="icon" className={"h-6 w-6 " + className} />;
+    return (
+      <Image
+        src={iconUri}
+        alt="icon"
+        width={24}
+        height={24}
+        className={className}
+      />
+    );
   } else if (resolvedTheme === "dark") {
     const darkUri = uri + "-dark" + extension;
-    return <img src={darkUri} alt="icon" className={"h-6 w-6 " + className} />;
+    return (
+      <Image
+        src={darkUri}
+        alt="icon"
+        width={24}
+        height={24}
+        className={className}
+      />
+    );
   }
 
   const lightUri = uri + "-light" + extension;
-  return <img src={lightUri} alt="icon" className={"h-6 w-6 " + className} />;
+  return (
+    <Image
+      src={lightUri}
+      alt="icon"
+      width={24}
+      height={24}
+      className={className}
+    />
+  );
 }

web/components/modals/extension-modal.tsx

@@ -81,7 +81,7 @@ export default function ExtensionModal({
                 description: ext.description ?? "No description available",
                 displayName: ext.displayName ?? ext.name,
               },
-              isEnabled: false,
+              isEnabled: true,
               remoteOrigin: `https://cdn.pulse-editor.com/extension`,
             };
           });
@@ -227,14 +227,16 @@ function ExtensionPreview({
 
   const editorContext = useContext(EditorContext);
 
+  const [isShowInfo, setIsShowInfo] = useState(false);
+
   useEffect(() => {
-    setIsEnabled(extension.isEnabled);
     setIsLoaded(true);
-    setIsInstalled(
-      editorContext?.persistSettings?.extensions?.some(
-        (ext) => ext.config.id === extension.config.id,
-      ) ?? false,
+
+    const foundExt = editorContext?.persistSettings?.extensions?.find(
+      (ext) => ext.config.id === extension.config.id,
     );
+    setIsInstalled(foundExt !== undefined);
+    setIsEnabled(foundExt?.isEnabled ?? false);
   }, [extension]);
 
   function toggleExtension() {
@@ -255,7 +257,16 @@ function ExtensionPreview({
 
   return (
     <div className="w-full">
-      <div className="relative h-28 w-full">
+      <div
+        className="relative h-28 w-full"
+        onMouseEnter={() => {
+          setIsShowInfo(true);
+        }}
+        // Hide show info when user taps outside of the modal
+        onMouseLeave={() => {
+          setIsShowInfo(false);
+        }}
+      >
         <div className="absolute top-0 right-0.5 z-10">
           <div className="flex flex-col">
             {showInstalledChip && isInstalled && (
@@ -269,7 +280,7 @@ function ExtensionPreview({
           </div>
         </div>
         <Button
-          className="m-0 h-full w-full rounded-md p-0"
+          className="relative m-0 h-full w-full rounded-md p-0"
           onContextMenu={(e) => {
             e.preventDefault();
             // Get parent element position
@@ -284,6 +295,41 @@ function ExtensionPreview({
             }));
           }}
         ></Button>
+        {isShowInfo && (
+          <div className="absolute bottom-0.5 left-1/2 flex w-full -translate-x-1/2 justify-center gap-x-0.5">
+            <Button color="secondary" size="sm">
+              Details
+            </Button>
+            {!isInstalled ? (
+              <Button
+                color="primary"
+                size="sm"
+                onPress={(e) => {
+                  installExtension(extension).then(() => {
+                    toast.success("Extension installed");
+                    setIsInstalled(true);
+                    setIsEnabled(extension.isEnabled);
+                  });
+                }}
+              >
+                Install
+              </Button>
+            ) : (
+              <Button
+                color="danger"
+                size="sm"
+                onPress={(e) => {
+                  uninstallExtension(extension.config.id).then(() => {
+                    toast.success("Extension uninstalled");
+                    setIsInstalled(false);
+                  });
+                }}
+              >
+                Uninstall
+              </Button>
+            )}
+          </div>
+        )}
         <ContextMenu state={contextMenuState} setState={setContextMenuState}>
           <div className="flex flex-col">
             {isInstalled ? (
@@ -307,6 +353,7 @@ function ExtensionPreview({
                   installExtension(extension).then(() => {
                     toast.success("Extension installed");
                     setIsInstalled(true);
+                    setIsEnabled(extension.isEnabled);
                   });
                   setContextMenuState({ x: 0, y: 0, isOpen: false });
                 }}