fix(dashboard): rewrite drag-to-track with Pointer Events and pointer capture

alireza787b · claude · alireza787b · commit bad344b40db0 · 2026-02-15T17:23:37.000+08:00
Classic mode bounding box drawing was unreliable — the rectangle stopped
tracking when the cursor left the video container, coordinates were
sub-pixel (blurry), and the overlay had no z-index (could render behind
the video canvas).

Changes:
- useBoundingBoxHandlers: Replace separate mouse + touch handlers with
  unified Pointer Events. Use setPointerCapture() so drag continues even
  when pointer leaves the container. Clamp coordinates to container
  bounds. Round to integer pixels for crisp rendering.
- BoundingBoxDrawer: Wire onPointerDown/Move/Up instead of 6 separate
  mouse/touch handlers. Add z-index: 5 to bbox overlay. Add userSelect:
  none to prevent text selection during drag. Use 2px border with subtle
  fill for cleaner visual.
- DashboardPage: Update destructured props to match new API.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/dashboard/src/components/BoundingBoxDrawer.js b/dashboard/src/components/BoundingBoxDrawer.js
@@ -23,12 +23,9 @@ const BoundingBoxDrawer = ({
   startPos,
   currentPos,
   boundingBox,
-  handleMouseDown,
-  handleMouseMove,
-  handleMouseUp,
-  handleTouchStart,
-  handleTouchMove,
-  handleTouchEnd,
+  handlePointerDown,
+  handlePointerMove,
+  handlePointerUp,
   videoSrc,
   protocol,
   smartModeActive,
@@ -68,13 +65,13 @@ const BoundingBoxDrawer = ({
 
   const isDrawing = (startPos && currentPos) || boundingBox;
 
-  let left, top, width, height;
+  let left = 0, top = 0, width = 0, height = 0;
 
   if (isDrawing && containerDimensions.width && containerDimensions.height) {
-    left = boundingBox ? boundingBox.left : Math.min(startPos.x, currentPos.x);
-    top = boundingBox ? boundingBox.top : Math.min(startPos.y, currentPos.y);
-    width = boundingBox ? boundingBox.width : Math.abs(currentPos.x - startPos.x);
-    height = boundingBox ? boundingBox.height : Math.abs(currentPos.y - startPos.y);
+    left = boundingBox ? boundingBox.left : Math.round(Math.min(startPos.x, currentPos.x));
+    top = boundingBox ? boundingBox.top : Math.round(Math.min(startPos.y, currentPos.y));
+    width = boundingBox ? boundingBox.width : Math.round(Math.abs(currentPos.x - startPos.x));
+    height = boundingBox ? boundingBox.height : Math.round(Math.abs(currentPos.y - startPos.y));
 
     const containerWidth = containerDimensions.width;
     const containerHeight = containerDimensions.height;
@@ -113,15 +110,14 @@ const BoundingBoxDrawer = ({
         display: 'block',
         width: '100%',
         touchAction: 'none',
+        userSelect: 'none',
+        WebkitUserSelect: 'none',
         cursor: smartModeActive ? 'crosshair' : (startPos ? 'crosshair' : 'cell'),
       }}
-      onMouseDown={!smartModeActive ? handleMouseDown : null}
-      onMouseMove={!smartModeActive ? handleMouseMove : null}
-      onMouseUp={!smartModeActive ? handleMouseUp : null}
-      onTouchStart={!smartModeActive ? handleTouchStart : null}
-      onTouchMove={!smartModeActive ? handleTouchMove : null}
-      onTouchEnd={!smartModeActive ? handleTouchEnd : null}
-      onClick={smartModeActive ? handleSmartClick : null}
+      onPointerDown={!smartModeActive ? handlePointerDown : undefined}
+      onPointerMove={!smartModeActive ? handlePointerMove : undefined}
+      onPointerUp={!smartModeActive ? handlePointerUp : undefined}
+      onClick={smartModeActive ? handleSmartClick : undefined}
     >
       <VideoStream protocol={protocol} src={videoSrc} />
 
@@ -180,14 +176,16 @@ const BoundingBoxDrawer = ({
         <div
           style={{
             position: 'absolute',
-            border: '3px solid #ff5722',
-            borderRadius: '2px',
-            left: left,
-            top: top,
-            width: width,
-            height: height,
+            left,
+            top,
+            width,
+            height,
+            border: '2px solid #ff5722',
+            borderRadius: 1,
+            backgroundColor: 'rgba(255, 87, 34, 0.08)',
             pointerEvents: 'none',
-            boxShadow: '0 0 0 2px rgba(255, 87, 34, 0.2), 0 0 8px rgba(255, 87, 34, 0.4)',
+            zIndex: 5,
+            boxShadow: '0 0 0 1px rgba(255, 87, 34, 0.3)',
           }}
         />
       )}
diff --git a/dashboard/src/hooks/useBoundingBoxHandlers.js b/dashboard/src/hooks/useBoundingBoxHandlers.js
@@ -1,22 +1,27 @@
-import { useState, useRef, useEffect } from 'react';
+import { useState, useRef, useEffect, useCallback } from 'react';
 import { endpoints } from '../services/apiEndpoints';
 
+/**
+ * Hook for bounding-box drag drawing (classic tracker) and smart-click (AI tracker).
+ *
+ * Uses Pointer Events with pointer capture so the rectangle tracks cleanly
+ * even when the cursor/finger leaves the video container during a drag.
+ */
 const useBoundingBoxHandlers = (isTracking, setIsTracking, smartModeActive = false) => {
   const [startPos, setStartPos] = useState(null);
   const [currentPos, setCurrentPos] = useState(null);
   const [boundingBox, setBoundingBox] = useState(null);
-  const imageRef = useRef();
+  const imageRef = useRef(null);
 
   const defaultBoundingBoxSize =
     parseFloat(process.env.REACT_APP_DEFAULT_BOUNDING_BOX_SIZE) || 0.2;
 
   const timeoutRef = useRef(null);
+  const draggingRef = useRef(false);
 
   useEffect(() => {
     return () => {
-      if (timeoutRef.current) {
-        clearTimeout(timeoutRef.current);
-      }
+      if (timeoutRef.current) clearTimeout(timeoutRef.current);
     };
   }, []);
 
@@ -26,7 +31,7 @@ const useBoundingBoxHandlers = (isTracking, setIsTracking, smartModeActive = fal
     return Math.sqrt(dx * dx + dy * dy);
   };
 
-  const startTracking = async (bbox) => {
+  const startTracking = useCallback(async (bbox) => {
     try {
       if (isTracking) {
         await fetch(endpoints.stopTracking, {
@@ -46,60 +51,65 @@ const useBoundingBoxHandlers = (isTracking, setIsTracking, smartModeActive = fal
     } catch (error) {
       console.error('Error:', error);
     }
-  };
+  }, [isTracking, setIsTracking]);
 
-  const sendSmartClick = async (normX, normY) => {
-    try {
-      const res = await fetch(endpoints.smartClick, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ x: normX, y: normY }),
-      });
-      const data = await res.json();
-      console.log('Smart click sent:', data);
-    } catch (err) {
-      console.error('Failed to send smart click:', err);
-    }
-  };
+  // ── Pointer handlers (unified mouse + touch + pen) ─────────────────
+
+  const handlePointerDown = useCallback((e) => {
+    if (smartModeActive) return;       // smart mode uses onClick instead
+    if (e.button !== 0) return;        // left button only
+
+    e.preventDefault();
+    e.target.setPointerCapture(e.pointerId);
+    draggingRef.current = true;
 
-  const handleStart = (clientX, clientY) => {
     const rect = imageRef.current.getBoundingClientRect();
-    const x = clientX - rect.left;
-    const y = clientY - rect.top;
-
-    if (smartModeActive) {
-      // Send normalized smart click
-      const normX = x / rect.width;
-      const normY = y / rect.height;
-      sendSmartClick(normX, normY);
-      return;
-    }
+    const x = Math.round(e.clientX - rect.left);
+    const y = Math.round(e.clientY - rect.top);
 
     setStartPos({ x, y });
     setCurrentPos({ x, y });
     setBoundingBox(null);
-  };
+  }, [smartModeActive]);
+
+  const handlePointerMove = useCallback((e) => {
+    if (!draggingRef.current || smartModeActive) return;
+    if (!imageRef.current) return;
+
+    const rect = imageRef.current.getBoundingClientRect();
+    // Clamp to container bounds for clean drawing
+    const x = Math.round(Math.max(0, Math.min(e.clientX - rect.left, rect.width)));
+    const y = Math.round(Math.max(0, Math.min(e.clientY - rect.top, rect.height)));
+
+    setCurrentPos({ x, y });
+  }, [smartModeActive]);
+
+  const handlePointerUp = useCallback(async (e) => {
+    if (!draggingRef.current) return;
+    draggingRef.current = false;
 
-  const handleMove = (clientX, clientY) => {
-    if (startPos && !smartModeActive) {
-      const rect = imageRef.current.getBoundingClientRect();
-      const x = clientX - rect.left;
-      const y = clientY - rect.top;
-      setCurrentPos({ x, y });
+    if (e.target.hasPointerCapture(e.pointerId)) {
+      e.target.releasePointerCapture(e.pointerId);
     }
-  };
 
-  const handleEnd = async () => {
-    if (!startPos || !currentPos || smartModeActive) return;
+    // Read refs for the final computation
+    const start = startPos;
+    const current = currentPos;
+    if (!start || !current || smartModeActive) {
+      setStartPos(null);
+      setCurrentPos(null);
+      return;
+    }
 
     const rect = imageRef.current.getBoundingClientRect();
-    const distance = getDistance(startPos, currentPos);
+    const distance = getDistance(start, current);
     let bbox;
 
     const dragThreshold = Math.max(5, (window.devicePixelRatio || 1) * 5);
     if (distance < dragThreshold) {
-      const centerX = startPos.x;
-      const centerY = startPos.y;
+      // Click-to-center: create default-size box around click point
+      const centerX = start.x;
+      const centerY = start.y;
       const width = rect.width * defaultBoundingBoxSize;
       const height = rect.height * defaultBoundingBoxSize;
       const left = centerX - width / 2;
@@ -112,12 +122,18 @@ const useBoundingBoxHandlers = (isTracking, setIsTracking, smartModeActive = fal
         height: defaultBoundingBoxSize,
       };
 
-      setBoundingBox({ left, top, width, height });
+      setBoundingBox({
+        left: Math.round(left),
+        top: Math.round(top),
+        width: Math.round(width),
+        height: Math.round(height),
+      });
     } else {
-      const x1 = startPos.x / rect.width;
-      const y1 = startPos.y / rect.height;
-      const x2 = currentPos.x / rect.width;
-      const y2 = currentPos.y / rect.height;
+      // Drag: compute normalized bbox
+      const x1 = start.x / rect.width;
+      const y1 = start.y / rect.height;
+      const x2 = current.x / rect.width;
+      const y2 = current.y / rect.height;
 
       bbox = {
         x: Math.min(x1, x2),
@@ -127,10 +143,10 @@ const useBoundingBoxHandlers = (isTracking, setIsTracking, smartModeActive = fal
       };
 
       setBoundingBox({
-        left: Math.min(startPos.x, currentPos.x),
-        top: Math.min(startPos.y, currentPos.y),
-        width: Math.abs(currentPos.x - startPos.x),
-        height: Math.abs(currentPos.y - startPos.y),
+        left: Math.round(Math.min(start.x, current.x)),
+        top: Math.round(Math.min(start.y, current.y)),
+        width: Math.round(Math.abs(current.x - start.x)),
+        height: Math.round(Math.abs(current.y - start.y)),
       });
     }
 
@@ -143,38 +159,16 @@ const useBoundingBoxHandlers = (isTracking, setIsTracking, smartModeActive = fal
 
     setStartPos(null);
     setCurrentPos(null);
-  };
-
-  const handleMouseDown = (e) => handleStart(e.clientX, e.clientY);
-  const handleMouseMove = (e) => handleMove(e.clientX, e.clientY);
-  const handleMouseUp = handleEnd;
-
-  const handleTouchStart = (e) => {
-    e.preventDefault();
-    handleStart(e.touches[0].clientX, e.touches[0].clientY);
-  };
-
-  const handleTouchMove = (e) => {
-    e.preventDefault();
-    handleMove(e.touches[0].clientX, e.touches[0].clientY);
-  };
-
-  const handleTouchEnd = (e) => {
-    e.preventDefault();
-    handleEnd();
-  };
+  }, [startPos, currentPos, smartModeActive, defaultBoundingBoxSize, startTracking]);
 
   return {
     imageRef,
     startPos,
     currentPos,
     boundingBox,
-    handleMouseDown,
-    handleMouseMove,
-    handleMouseUp,
-    handleTouchStart,
-    handleTouchMove,
-    handleTouchEnd,
+    handlePointerDown,
+    handlePointerMove,
+    handlePointerUp,
   };
 };
 
diff --git a/dashboard/src/pages/DashboardPage.js b/dashboard/src/pages/DashboardPage.js
@@ -88,12 +88,9 @@ const DashboardPage = () => {
     startPos,
     currentPos,
     boundingBox,
-    handleMouseDown,
-    handleMouseMove,
-    handleMouseUp,
-    handleTouchStart,
-    handleTouchMove,
-    handleTouchEnd,
+    handlePointerDown,
+    handlePointerMove,
+    handlePointerUp,
   } = useBoundingBoxHandlers(isTracking, setIsTracking, smartModeActive);
 
   const handleTrackingToggle = async () => {
@@ -294,12 +291,9 @@ const DashboardPage = () => {
                             startPos={startPos}
                             currentPos={currentPos}
                             boundingBox={boundingBox}
-                            handleMouseDown={handleMouseDown}
-                            handleMouseMove={handleMouseMove}
-                            handleMouseUp={handleMouseUp}
-                            handleTouchStart={handleTouchStart}
-                            handleTouchMove={handleTouchMove}
-                            handleTouchEnd={handleTouchEnd}
+                            handlePointerDown={handlePointerDown}
+                            handlePointerMove={handlePointerMove}
+                            handlePointerUp={handlePointerUp}
                             videoSrc={videoFeed}
                             protocol={streamingProtocol}
                             smartModeActive={smartModeActive}
