feat: relative mouse (#246)

ym · web-flow · commit d52e7d04d12f · 2025-03-19T11:47:15.000+01:00
diff --git a/jsonrpc.go b/jsonrpc.go
@@ -799,6 +799,7 @@ var rpcHandlers = map[string]RPCHandler{
 	"getCloudState":          {Func: rpcGetCloudState},
 	"keyboardReport":         {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}},
 	"absMouseReport":         {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}},
+	"relMouseReport":         {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}},
 	"wheelReport":            {Func: rpcWheelReport, Params: []string{"wheelY"}},
 	"getVideoState":          {Func: rpcGetVideoState},
 	"getUSBState":            {Func: rpcGetUSBState},
diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx
@@ -14,6 +14,7 @@ export default function InfoBar() {
   const activeModifiers = useHidStore(state => state.activeModifiers);
   const mouseX = useMouseStore(state => state.mouseX);
   const mouseY = useMouseStore(state => state.mouseY);
+  const mouseMove = useMouseStore(state => state.mouseMove);
 
   const videoClientSize = useVideoStore(
     state => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`,
@@ -62,7 +63,7 @@ export default function InfoBar() {
               </div>
             ) : null}
 
-            {settings.debugMode ? (
+            {(settings.debugMode && settings.mouseMode == "absolute") ? (
               <div className="flex w-[118px] items-center gap-x-1">
                 <span className="text-xs font-semibold">Pointer:</span>
                 <span className="text-xs">
@@ -71,6 +72,17 @@ export default function InfoBar() {
               </div>
             ) : null}
 
+            {(settings.debugMode && settings.mouseMode == "relative") ? (
+              <div className="flex w-[118px] items-center gap-x-1">
+                <span className="text-xs font-semibold">Last Move:</span>
+                <span className="text-xs">
+                  {mouseMove ?
+                    `${mouseMove.x},${mouseMove.y} ${mouseMove.buttons ? `(${mouseMove.buttons})` : ""}` :
+                    "N/A"}
+                </span>
+              </div>
+            ) : null}
+
             {settings.debugMode && (
               <div className="flex w-[156px] items-center gap-x-1">
                 <span className="text-xs font-semibold">USB State:</span>
diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx
@@ -29,6 +29,7 @@ export default function WebRTCVideo() {
   const settings = useSettingsStore();
   const { sendKeyboardEvent, resetKeyboardState } = useKeyboard();
   const setMousePosition = useMouseStore(state => state.setMousePosition);
+  const setMouseMove = useMouseStore(state => state.setMouseMove);
   const {
     setClientSize: setVideoClientSize,
     setSize: setVideoSize,
@@ -93,19 +94,44 @@ export default function WebRTCVideo() {
   );
 
   // Mouse-related
-  const sendMouseMovement = useCallback(
+  const calcDelta = (pos: number) => Math.abs(pos) < 10 ? pos * 2 : pos;
+  const sendRelMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
-      send("absMouseReport", { x, y, buttons });
+      if (settings.mouseMode !== "relative") return;
+      // if we ignore the event, double-click will not work
+      // if (x === 0 && y === 0 && buttons === 0) return;
+      send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons });
+      setMouseMove({ x, y, buttons });
+    },
+    [send, setMouseMove, settings.mouseMode],
+  );
 
+  const relMouseMoveHandler = useCallback(
+    (e: MouseEvent) => {
+      if (settings.mouseMode !== "relative") return;
+
+      // Send mouse movement
+      const { buttons } = e;
+      sendRelMouseMovement(e.movementX, e.movementY, buttons);
+    },
+    [sendRelMouseMovement, settings.mouseMode],
+  );
+
+  const sendAbsMouseMovement = useCallback(
+    (x: number, y: number, buttons: number) => {
+      if (settings.mouseMode !== "absolute") return;
+      send("absMouseReport", { x, y, buttons });
       // We set that for the debug info bar
       setMousePosition(x, y);
     },
-    [send, setMousePosition],
+    [send, setMousePosition, settings.mouseMode],
   );
 
-  const mouseMoveHandler = useCallback(
+  const absMouseMoveHandler = useCallback(
     (e: MouseEvent) => {
       if (!videoClientWidth || !videoClientHeight) return;
+      if (settings.mouseMode !== "absolute") return;
+
       // Get the aspect ratios of the video element and the video stream
       const videoElementAspectRatio = videoClientWidth / videoClientHeight;
       const videoStreamAspectRatio = videoWidth / videoHeight;
@@ -140,9 +166,9 @@ export default function WebRTCVideo() {
 
       // Send mouse movement
       const { buttons } = e;
-      sendMouseMovement(x, y, buttons);
+      sendAbsMouseMovement(x, y, buttons);
     },
-    [sendMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight],
+    [sendAbsMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight, settings.mouseMode],
   );
 
   const trackpadSensitivity = useDeviceSettingsStore(state => state.trackpadSensitivity);
@@ -193,8 +219,8 @@ export default function WebRTCVideo() {
   );
 
   const resetMousePosition = useCallback(() => {
-    sendMouseMovement(0, 0, 0);
-  }, [sendMouseMovement]);
+    sendAbsMouseMovement(0, 0, 0);
+  }, [sendAbsMouseMovement]);
 
   // Keyboard-related
   const handleModifierKeys = useCallback(
@@ -371,9 +397,9 @@ export default function WebRTCVideo() {
       const abortController = new AbortController();
       const signal = abortController.signal;
 
-      videoElmRefValue.addEventListener("mousemove", mouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", mouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", mouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
       videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal });
       videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
         signal,
@@ -395,14 +421,39 @@ export default function WebRTCVideo() {
       };
     },
     [
-      mouseMoveHandler,
+      absMouseMoveHandler,
       resetMousePosition,
       onVideoPlaying,
       mouseWheelHandler,
       videoKeyUpHandler,
     ],
   );
 
+  useEffect(
+    function setupRelativeMouseEventListeners() {
+      if (settings.mouseMode !== "relative") return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      // bind to body to capture all mouse events
+      const body = document.querySelector("body");
+      if (!body) return;
+
+      body.addEventListener("mousemove", relMouseMoveHandler, { signal });
+      body.addEventListener("pointerdown", relMouseMoveHandler, { signal });
+      body.addEventListener("pointerup", relMouseMoveHandler, { signal });
+
+      return () => {
+        abortController.abort();
+
+        body.removeEventListener("mousemove", relMouseMoveHandler);
+        body.removeEventListener("pointerdown", relMouseMoveHandler);
+        body.removeEventListener("pointerup", relMouseMoveHandler);
+      };
+    }, [settings.mouseMode, relMouseMoveHandler],
+  )
+
   useEffect(
     function updateVideoStream() {
       if (!mediaStream) return;
diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts
@@ -197,15 +197,23 @@ export const useRTCStore = create<RTCState>(set => ({
   setTerminalChannel: channel => set({ terminalChannel: channel }),
 }));
 
+interface MouseMove {
+  x: number;
+  y: number;
+  buttons: number;
+}
 interface MouseState {
   mouseX: number;
   mouseY: number;
+  mouseMove?: MouseMove;
+  setMouseMove: (move?: MouseMove) => void;
   setMousePosition: (x: number, y: number) => void;
 }
 
 export const useMouseStore = create<MouseState>(set => ({
   mouseX: 0,
   mouseY: 0,
+  setMouseMove: (move?: MouseMove) => set({ mouseMove: move }),
   setMousePosition: (x, y) => set({ mouseX: x, mouseY: y }),
 }));
 
@@ -543,12 +551,12 @@ export interface UpdateState {
   setOtaState: (state: UpdateState["otaState"]) => void;
   setUpdateDialogHasBeenMinimized: (hasBeenMinimized: boolean) => void;
   modalView:
-    | "loading"
-    | "updating"
-    | "upToDate"
-    | "updateAvailable"
-    | "updateCompleted"
-    | "error";
+  | "loading"
+  | "updating"
+  | "upToDate"
+  | "updateAvailable"
+  | "updateCompleted"
+  | "error";
   setModalView: (view: UpdateState["modalView"]) => void;
   setUpdateErrorMessage: (errorMessage: string) => void;
   updateErrorMessage: string | null;
@@ -612,12 +620,12 @@ export const useUsbConfigModalStore = create<UsbConfigModalState>(set => ({
 
 interface LocalAuthModalState {
   modalView:
-    | "createPassword"
-    | "deletePassword"
-    | "updatePassword"
-    | "creationSuccess"
-    | "deleteSuccess"
-    | "updateSuccess";
+  | "createPassword"
+  | "deletePassword"
+  | "updatePassword"
+  | "creationSuccess"
+  | "deleteSuccess"
+  | "updateSuccess";
   setModalView: (view: LocalAuthModalState["modalView"]) => void;
 }
 
diff --git a/ui/src/routes/devices.$id.settings.mouse.tsx b/ui/src/routes/devices.$id.settings.mouse.tsx
@@ -1,23 +1,27 @@
-import { SettingsPageHeader } from "@components/SettingsPageheader";
-import { SettingsItem } from "./devices.$id.settings";
-import { Checkbox } from "@/components/Checkbox";
-import { GridCard } from "@/components/Card";
+import MouseIcon from "@/assets/mouse-icon.svg";
 import PointingFinger from "@/assets/pointing-finger.svg";
-import { CheckCircleIcon } from "@heroicons/react/16/solid";
+import { GridCard } from "@/components/Card";
+import { Checkbox } from "@/components/Checkbox";
 import { useDeviceSettingsStore, useSettingsStore } from "@/hooks/stores";
+import { useJsonRpc } from "@/hooks/useJsonRpc";
 import notifications from "@/notifications";
+import { SettingsPageHeader } from "@components/SettingsPageheader";
+import { CheckCircleIcon } from "@heroicons/react/16/solid";
 import { useCallback, useEffect, useState } from "react";
-import { useJsonRpc } from "@/hooks/useJsonRpc";
-import { cx } from "../cva.config";
+import { FeatureFlag } from "../components/FeatureFlag";
 import { SelectMenuBasic } from "../components/SelectMenuBasic";
 import { useFeatureFlag } from "../hooks/useFeatureFlag";
-import { FeatureFlag } from "../components/FeatureFlag";
+import { SettingsItem } from "./devices.$id.settings";
 
 type ScrollSensitivity = "low" | "default" | "high";
 
 export default function SettingsKeyboardMouseRoute() {
   const hideCursor = useSettingsStore(state => state.isCursorHidden);
   const setHideCursor = useSettingsStore(state => state.setCursorVisibility);
+
+  const mouseMode = useSettingsStore(state => state.mouseMode);
+  const setMouseMode = useSettingsStore(state => state.setMouseMode);
+
   const scrollSensitivity = useDeviceSettingsStore(state => state.scrollSensitivity);
   const setScrollSensitivity = useDeviceSettingsStore(
     state => state.setScrollSensitivity,
@@ -122,19 +126,19 @@ export default function SettingsKeyboardMouseRoute() {
         </SettingsItem>
         <div className="space-y-4">
           <SettingsItem title="Modes" description="Choose the mouse input mode" />
-          <div className="flex flex-col items-center gap-4 md:flex-row">
+          <div className="flex items-center gap-4">
             <button
-              className="group block w-full grow"
-              onClick={() => console.log("Absolute mouse mode clicked")}
+              className="block group grow"
+              onClick={() => { setMouseMode("absolute"); }}
             >
               <GridCard>
-                <div className="group flex items-center gap-x-4 px-4 py-3">
+                <div className="flex items-center px-4 py-3 group gap-x-4">
                   <img
                     className="w-6 shrink-0 dark:invert"
                     src={PointingFinger}
                     alt="Finger touching a screen"
                   />
-                  <div className="flex grow items-center justify-between">
+                  <div className="flex items-center justify-between grow">
                     <div className="text-left">
                       <h3 className="text-sm font-semibold text-black dark:text-white">
                         Absolute
@@ -143,41 +147,32 @@ export default function SettingsKeyboardMouseRoute() {
                         Most convenient
                       </p>
                     </div>
-                    <CheckCircleIcon
-                      className={cx(
-                        "h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
-                      )}
-                    />
+                    {mouseMode === "absolute" && (
+                      <CheckCircleIcon className="w-4 h-4 text-blue-700 dark:text-blue-500" />
+                    )}
                   </div>
                 </div>
               </GridCard>
             </button>
             <button
-              className="group block w-full grow cursor-not-allowed opacity-50"
-              disabled
+              className="block group grow"
+              onClick={() => { setMouseMode("relative"); }}
             >
               <GridCard>
-                <div className="group flex items-center gap-x-4 px-4 py-3">
-                  <img
-                    className="w-6 shrink-0 dark:invert"
-                    src={PointingFinger}
-                    alt="Finger touching a screen"
-                  />
-                  <div className="flex grow items-center justify-between">
+                <div className="flex items-center px-4 py-3 gap-x-4">
+                  <img className="w-6 shrink-0 dark:invert" src={MouseIcon} alt="Mouse icon" />
+                  <div className="flex items-center justify-between grow">
                     <div className="text-left">
                       <h3 className="text-sm font-semibold text-black dark:text-white">
                         Relative
                       </h3>
                       <p className="text-xs leading-none text-slate-800 dark:text-slate-300">
-                        Most Compatible
+                        Most Compatible (Beta)
                       </p>
                     </div>
-                    <CheckCircleIcon
-                      className={cx(
-                        "hidden",
-                        "h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
-                      )}
-                    />
+                    {mouseMode === "relative" && (
+                      <CheckCircleIcon className="w-4 h-4 text-blue-700 dark:text-blue-500" />
+                    )}
                   </div>
                 </div>
               </GridCard>
diff --git a/usb.go b/usb.go
@@ -1,8 +1,9 @@
 package kvm
 
 import (
-	"github.com/jetkvm/kvm/internal/usbgadget"
 	"time"
+
+	"github.com/jetkvm/kvm/internal/usbgadget"
 )
 
 var gadget *usbgadget.UsbGadget
@@ -33,6 +34,10 @@ func rpcAbsMouseReport(x, y int, buttons uint8) error {
 	return gadget.AbsMouseReport(x, y, buttons)
 }
 
+func rpcRelMouseReport(dx, dy int8, buttons uint8) error {
+	return gadget.RelMouseReport(dx, dy, buttons)
+}
+
 func rpcWheelReport(wheelY int8) error {
 	return gadget.AbsMouseWheelReport(wheelY)
 }

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,9 @@`
`1`	`1`	`package kvm`
`2`	`2`
`3`	`3`	`import (`
`4`		`- "github.com/jetkvm/kvm/internal/usbgadget"`
`5`	`4`	`"time"`
	`5`	`+`
	`6`	`+ "github.com/jetkvm/kvm/internal/usbgadget"`
`6`	`7`	`)`
`7`	`8`
`8`	`9`	`var gadget *usbgadget.UsbGadget`
`@@ -33,6 +34,10 @@ func rpcAbsMouseReport(x, y int, buttons uint8) error {`
`33`	`34`	`return gadget.AbsMouseReport(x, y, buttons)`
`34`	`35`	`}`
`35`	`36`
	`37`	`+func rpcRelMouseReport(dx, dy int8, buttons uint8) error {`
	`38`	`+ return gadget.RelMouseReport(dx, dy, buttons)`
	`39`	`+}`
	`40`	`+`
`36`	`41`	`func rpcWheelReport(wheelY int8) error {`
`37`	`42`	`return gadget.AbsMouseWheelReport(wheelY)`
`38`	`43`	`}`