feat: ocr

Author: ym

ui/package.json

@@ -4,7 +4,7 @@
   "version": "2025.09.03.2100",
   "type": "module",
   "engines": {
-    "node": "22.15.0"
+    "node": "^22.15.0"
   },
   "scripts": {
     "dev": "./dev_device.sh",
@@ -78,6 +78,7 @@
     "prettier": "^3.6.2",
     "prettier-plugin-tailwindcss": "^0.6.14",
     "tailwindcss": "^4.1.12",
+    "tesseract.js": "6.0.1",
     "typescript": "^5.9.2",
     "vite": "^7.1.4",
     "vite-tsconfig-paths": "^5.1.4"

ui/src/components/ActionBar.tsx

@@ -1,4 +1,4 @@
-import { MdOutlineContentPasteGo } from "react-icons/md";
+import { MdOutlineContentPasteGo, MdOutlineDocumentScanner } from "react-icons/md";
 import { LuCable, LuHardDrive, LuMaximize, LuSettings, LuSignal } from "react-icons/lu";
 import { FaKeyboard } from "react-icons/fa6";
 import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
@@ -19,11 +19,14 @@ import WakeOnLanModal from "@/components/popovers/WakeOnLan/Index";
 import MountPopopover from "@/components/popovers/MountPopover";
 import ExtensionPopover from "@/components/popovers/ExtensionPopover";
 import { useDeviceUiNavigation } from "@/hooks/useAppNavigation";
+import OCRModal from "./popovers/OCRModal";
 
 export default function Actionbar({
   requestFullscreen,
+  videoElmRef,
 }: {
   requestFullscreen: () => Promise<void>;
+  videoElmRef?: React.RefObject<HTMLVideoElement | null>;
 }) {
   const { navigateTo } = useDeviceUiNavigation();
   const { isVirtualKeyboardEnabled, setVirtualKeyboardEnabled } = useHidStore();
@@ -99,6 +102,36 @@ export default function Actionbar({
               }}
             </PopoverPanel>
           </Popover>
+          <Popover>
+            <PopoverButton as={Fragment}>
+              <Button
+                size="XS"
+                theme="light"
+                text="OCR"
+                LeadingIcon={MdOutlineDocumentScanner}
+                onClick={() => {
+                  setDisableVideoFocusTrap(true);
+                }}
+              />
+            </PopoverButton>
+            <PopoverPanel
+              anchor="bottom start"
+              transition
+              className={cx(
+                "z-10 flex w-[420px] origin-top flex-col overflow-visible!",
+                "flex origin-top flex-col transition duration-300 ease-out data-closed:translate-y-8 data-closed:opacity-0",
+              )}
+            >
+              {({ open }) => {
+                checkIfStateChanged(open);
+                return (
+                  <div className="mx-auto w-full max-w-xl">
+                    <OCRModal videoElmRef={videoElmRef} />
+                  </div>
+                );
+              }}
+            </PopoverPanel>
+          </Popover>
           <div className="relative">
             <Popover>
               <PopoverButton as={Fragment}>

ui/src/components/WebRTCVideo.tsx

@@ -487,7 +487,7 @@ export default function WebRTCVideo() {
             disabled={peerConnection?.connectionState !== "connected"}
             className="contents"
           >
-            <Actionbar requestFullscreen={requestFullscreen} />
+            <Actionbar requestFullscreen={requestFullscreen} videoElmRef={videoElm} />
             <MacroBar />
           </fieldset>
         </div>

ui/src/hooks/useOCR.ts

@@ -0,0 +1,23 @@
+import { type WorkerOptions } from "tesseract.js";
+
+export type ImageLike = string | HTMLImageElement | HTMLCanvasElement | HTMLVideoElement
+  | CanvasRenderingContext2D | File | Blob | OffscreenCanvas;
+
+// tesseract.js is h
+async function ocrImage(
+  language: string | string[],
+  image: ImageLike,
+  options?: Partial<WorkerOptions>,
+) {
+  const { createWorker } = await import('tesseract.js')
+  const worker = await createWorker(language, undefined, options)
+  const { data: { text } } = await worker.recognize(image)
+  await worker.terminate()
+  return text
+}
+
+export default function useOCR() {
+  return {
+    ocrImage,
+  }
+}

ui/vite.config.ts

@@ -31,20 +31,30 @@ export default defineConfig(({ mode, command }) => {
     esbuild: {
       pure: ["console.debug"],
     },
-    build: { outDir: isCloud ? "dist" : "../static" },
+    build: {
+      outDir: isCloud ? "dist" : "../static",
+      rollupOptions: {
+        external: ["tesseract.js"],
+        output: {
+          paths: {
+            "tesseract.js": "https://cdn.jsdelivr.net/npm/[email protected]/dist/tesseract.esm.min.js",
+          },
+        },
+      },
+    },
     server: {
       host: "0.0.0.0",
       https: useSSL,
       proxy: JETKVM_PROXY_URL
         ? {
-            "/me": JETKVM_PROXY_URL,
-            "/device": JETKVM_PROXY_URL,
-            "/webrtc": JETKVM_PROXY_URL,
-            "/auth": JETKVM_PROXY_URL,
-            "/storage": JETKVM_PROXY_URL,
-            "/cloud": JETKVM_PROXY_URL,
-            "/developer": JETKVM_PROXY_URL,
-          }
+          "/me": JETKVM_PROXY_URL,
+          "/device": JETKVM_PROXY_URL,
+          "/webrtc": JETKVM_PROXY_URL,
+          "/auth": JETKVM_PROXY_URL,
+          "/storage": JETKVM_PROXY_URL,
+          "/cloud": JETKVM_PROXY_URL,
+          "/developer": JETKVM_PROXY_URL,
+        }
         : undefined,
     },
     base: onDevice && command === "build" ? "/static" : "/",