Add image gen and video gen options & add relevant react-api hooks

Author: Shellishack

.changeset/lazy-zebras-mate.md

@@ -0,0 +1,6 @@
+---
+"@pulse-editor/shared-utils": patch
+"@pulse-editor/react-api": patch
+---
+
+Add useLoading, image gen, video gen hooks

.changeset/pre.json

@@ -22,6 +22,7 @@
     "early-pumas-listen",
     "hot-symbols-fry",
     "large-moose-tap",
+    "lazy-zebras-mate",
     "mighty-ghosts-crash",
     "real-knives-rest",
     "rude-ducks-design",

npm-packages/react-api/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @pulse-editor/react-api
 
+## 0.1.1-alpha.17
+
+### Patch Changes
+
+- Add useLoading, image gen, video gen hooks
+- Updated dependencies
+  - @pulse-editor/shared-utils@0.1.1-alpha.17
+
 ## 0.1.1-alpha.16
 
 ### Patch Changes

npm-packages/react-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pulse-editor/react-api",
-  "version": "0.1.1-alpha.16",
+  "version": "0.1.1-alpha.17",
   "main": "dist/main.js",
   "files": [
     "dist"
@@ -38,7 +38,7 @@
     "typescript-eslint": "^8.30.1"
   },
   "peerDependencies": {
-    "@pulse-editor/shared-utils": "0.1.1-alpha.16",
+    "@pulse-editor/shared-utils": "0.1.1-alpha.17",
     "react": "^19.0.0",
     "react-dom": "^19.0.0"
   }

npm-packages/react-api/src/hooks/editor/use-file-view.ts

@@ -8,7 +8,6 @@ import useIMC from "../../lib/use-imc";
 
 export default function useFileView() {
   const [viewModel, setViewModel] = useState<ViewModel | undefined>(undefined);
-  const [isLoaded, setIsLoaded] = useState(false);
 
   const receiverHandlerMap = new Map<
     IMCMessageTypeEnum,
@@ -25,12 +24,6 @@ export default function useFileView() {
     }
   }, [isReady]);
 
-  useEffect(() => {
-    if (isLoaded) {
-      imc?.sendMessage(IMCMessageTypeEnum.Loaded);
-    }
-  }, [isLoaded, imc]);
-
   function updateViewModel(viewModel: ViewModel) {
     // sender.sendMessage(ViewBoxMessageTypeEnum.ViewFile, JSON.stringify(file));
     imc?.sendMessage(IMCMessageTypeEnum.WriteViewFile, viewModel);
@@ -39,6 +32,5 @@ export default function useFileView() {
   return {
     viewModel,
     updateViewModel,
-    setIsLoaded,
   };
 }

npm-packages/react-api/src/hooks/editor/use-loading.ts

@@ -0,0 +1,30 @@
+import { useEffect, useState } from "react";
+import useIMC from "../../lib/use-imc";
+import { IMCMessage, IMCMessageTypeEnum } from "@pulse-editor/shared-utils";
+
+export default function useLoading() {
+  const receiverHandlerMap = new Map<
+    IMCMessageTypeEnum,
+    (senderWindow: Window, message: IMCMessage) => Promise<void>
+  >();
+
+  const { imc, isReady } = useIMC(receiverHandlerMap);
+  const [isLoading, setIsLoading] = useState(true);
+
+  useEffect(() => {
+    if (isReady) {
+      imc?.sendMessage(IMCMessageTypeEnum.UseLoading, {
+        isLoading,
+      });
+    }
+  }, [isLoading]);
+
+  function toggleLoading(isLoading: boolean) {
+    setIsLoading((prev) => isLoading);
+  }
+
+  return {
+    isReady,
+    toggleLoading,
+  };
+}

npm-packages/react-api/src/hooks/modality/use-diffusion.ts

@@ -0,0 +1,54 @@
+import {
+  ImageModelConfig,
+  IMCMessage,
+  IMCMessageTypeEnum,
+} from "@pulse-editor/shared-utils";
+import useIMC from "../../lib/use-imc";
+
+export default function useImageGen() {
+  const receiverHandlerMap = new Map<
+    IMCMessageTypeEnum,
+    (senderWindow: Window, message: IMCMessage) => Promise<void>
+  >();
+
+  const { imc, isReady } = useIMC(receiverHandlerMap);
+
+  /**
+   *
+   * @param textPrompt The text prompt to generate the image.
+   * @param imagePrompt A URL to an image or an image in an ArrayBuffer.
+   * @param imageModelConfig (optional) The image model config to use.
+   * @returns The generated image in an ArrayBuffer.
+   */
+  async function runImageGen(
+    textPrompt?: string,
+    imagePrompt?: string | ArrayBuffer,
+    // LLM config is optional, if not provided, the default config will be used.
+    imageModelConfig?: ImageModelConfig
+  ): Promise<ArrayBuffer> {
+    if (!imc) {
+      throw new Error("IMC not initialized.");
+    }
+
+    if (!textPrompt && !imagePrompt) {
+      throw new Error("At least one of textPrompt or imagePrompt is required.");
+    }
+
+    const result = await imc
+      .sendMessage(IMCMessageTypeEnum.UseImageGen, {
+        textPrompt,
+        imagePrompt,
+        imageModelConfig,
+      })
+      .then((response) => {
+        return response as ArrayBuffer;
+      });
+
+    return result;
+  }
+
+  return {
+    runImageGen,
+    isReady,
+  };
+}

npm-packages/react-api/src/hooks/modality/use-image-gen.ts

@@ -0,0 +1,54 @@
+import {
+  IMCMessage,
+  IMCMessageTypeEnum,
+  VideoModelConfig,
+} from "@pulse-editor/shared-utils";
+import useIMC from "../../lib/use-imc";
+
+export default function useVideoGen() {
+  const receiverHandlerMap = new Map<
+    IMCMessageTypeEnum,
+    (senderWindow: Window, message: IMCMessage) => Promise<void>
+  >();
+
+  const { imc, isReady } = useIMC(receiverHandlerMap);
+
+  /**
+   *
+   * @param textPrompt The text prompt to generate the video.
+   * @param imagePrompt A URL to an image or an image in an ArrayBuffer.
+   * @param videoModelConfig (optional) The video model config to use.
+   * @returns The generated video in an ArrayBuffer.
+   */
+  async function runVideoGen(
+    textPrompt?: string,
+    imagePrompt?: string | ArrayBuffer,
+    // LLM config is optional, if not provided, the default config will be used.
+    videoModelConfig?: VideoModelConfig
+  ): Promise<ArrayBuffer> {
+    if (!imc) {
+      throw new Error("IMC not initialized.");
+    }
+
+    if (!textPrompt && !imagePrompt) {
+      throw new Error("At least one of textPrompt or imagePrompt is required.");
+    }
+
+    const result = await imc
+      .sendMessage(IMCMessageTypeEnum.UseVideoGen, {
+        textPrompt,
+        imagePrompt,
+        videoModelConfig,
+      })
+      .then((response) => {
+        return response as ArrayBuffer;
+      });
+
+    return result;
+  }
+
+  return {
+    runVideoGen,
+    isReady,
+  };
+}

npm-packages/react-api/src/hooks/modality/use-video-gen.ts

@@ -29,8 +29,6 @@ export default function useTerminal() {
 
         setWebsocketUrl(websocketUrl);
         setProjectHomePath(projectHomePath);
-
-        imc.sendMessage(IMCMessageTypeEnum.Loaded);
       });
     }
   }, [isReady]);