Use a shorter data window for recording and inference (#431)

We now record data for one second (990ms) and infer using time window while maintaining backwards compatibility with existing data samples that use the legacy window of 1800 ms. This makes the model feel much more responsive to user gesture/action changes.

- Look at the user's existing gestures to determine whether we should use the legacy data window
- Default to the new shorter data window where possible (new session, user deletes all recordings, user deletes all actions)
- Maintain existing data window for dataset.json and hex file loads

Other fixes and upgrades:
- Upgrade pxt-microbit-ml extension to include ZCR fix
- Fix prop casing for AlertIcon
- Remove circular reference in tour files
- Pull up listener for MakeCode project load so it affects all pages

Author: microbit-robert

src/App.tsx

@@ -1,12 +1,14 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { ChakraProvider } from "@chakra-ui/react";
+import { ChakraProvider, useToast } from "@chakra-ui/react";
 import { MakeCodeFrameDriver } from "@microbit/makecode-embed/react";
 import React, { ReactNode, useEffect, useMemo, useRef } from "react";
+import { useIntl } from "react-intl";
 import {
   Outlet,
   RouterProvider,
   ScrollRestoration,
   createBrowserRouter,
+  useNavigate,
 } from "react-router-dom";
 import { BufferedDataProvider } from "./buffered-data-hooks";
 import EditCodeDialog from "./components/EditCodeDialog";
@@ -20,20 +22,21 @@ import { deployment, useDeployment } from "./deployment";
 import { ProjectProvider } from "./hooks/project-hooks";
 import { LoggingProvider } from "./logging/logging-hooks";
 import TranslationProvider from "./messages/TranslationProvider";
+import CodePage from "./pages/CodePage";
 import DataSamplesPage from "./pages/DataSamplesPage";
 import HomePage from "./pages/HomePage";
 import ImportPage from "./pages/ImportPage";
 import NewPage from "./pages/NewPage";
 import TestingModelPage from "./pages/TestingModelPage";
+import { useStore } from "./store";
 import {
-  createDataSamplesPageUrl,
   createCodePageUrl,
+  createDataSamplesPageUrl,
   createHomePageUrl,
   createImportPageUrl,
   createNewPageUrl,
   createTestingModelPageUrl,
 } from "./urls";
-import CodePage from "./pages/CodePage";
 
 export interface ProviderLayoutProps {
   children: ReactNode;
@@ -69,6 +72,30 @@ const Providers = ({ children }: ProviderLayoutProps) => {
 
 const Layout = () => {
   const driverRef = useRef<MakeCodeFrameDriver>(null);
+  const navigate = useNavigate();
+  const toast = useToast();
+  const intl = useIntl();
+
+  useEffect(() => {
+    return useStore.subscribe(
+      (
+        { projectLoadTimestamp },
+        { projectLoadTimestamp: prevProjectLoadTimestamp }
+      ) => {
+        if (projectLoadTimestamp > prevProjectLoadTimestamp) {
+          // Side effects of loading a project, which MakeCode notifies us of.
+          navigate(createDataSamplesPageUrl());
+          toast({
+            position: "top",
+            duration: 5_000,
+            title: intl.formatMessage({ id: "project-loaded" }),
+            status: "info",
+          });
+        }
+      }
+    );
+  }, [intl, navigate, toast]);
+
   return (
     // We use this even though we have errorElement as this does logging.
     <ErrorBoundary>

src/buffered-data-hooks.tsx

@@ -1,9 +1,16 @@
 import { AccelerometerDataEvent } from "@microbit/microbit-connection";
-import { ReactNode, createContext, useContext, useEffect, useRef } from "react";
+import {
+  ReactNode,
+  createContext,
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+} from "react";
 import { BufferedData } from "./buffered-data";
 import { useConnectActions } from "./connect-actions-hooks";
 import { ConnectionStatus, useConnectStatus } from "./connect-status-hooks";
-import { mlSettings } from "./mlConfig";
+import { useStore } from "./store";
 
 const BufferedDataContext = createContext<BufferedData | null>(null);
 
@@ -31,14 +38,15 @@ export const useBufferedData = (): BufferedData => {
 const useBufferedDataInternal = (): BufferedData => {
   const [connectStatus] = useConnectStatus();
   const connection = useConnectActions();
+  const dataWindow = useStore((s) => s.dataWindow);
   const bufferRef = useRef<BufferedData>();
-  const getBuffer = () => {
+  const getBuffer = useCallback(() => {
     if (bufferRef.current) {
       return bufferRef.current;
     }
-    bufferRef.current = new BufferedData(mlSettings.numSamples * 2);
+    bufferRef.current = new BufferedData(dataWindow.minSamples * 2);
     return bufferRef.current;
-  };
+  }, [dataWindow.minSamples]);
   useEffect(() => {
     if (connectStatus !== ConnectionStatus.Connected) {
       return;
@@ -56,6 +64,6 @@ const useBufferedDataInternal = (): BufferedData => {
     return () => {
       connection.removeAccelerometerListener(listener);
     };
-  }, [connection, connectStatus]);
+  }, [connection, connectStatus, getBuffer]);
   return getBuffer();
 };

src/components/AlertIcon.tsx

@@ -5,7 +5,7 @@ interface AlertIconProps extends Omit<IconProps, "viewBox"> {}
 // Modified version of RiAlertFill from react-icons (MIT licensed icon set used elsewhere)
 const AlertIcon = ({ ...props }: AlertIconProps) => {
   return (
-    <Icon stroke-width="0" viewBox="0 0 24 24" {...props}>
+    <Icon strokeWidth="0" viewBox="0 0 24 24" {...props}>
       <path
         d="M12.8659 3.00017L22.3922 19.5002C22.6684 19.9785 22.5045 20.5901 22.0262 20.8662C21.8742 20.954 21.7017 21.0002 21.5262 21.0002H2.47363C1.92135 21.0002 1.47363 20.5525 1.47363 20.0002C1.47363 19.8246 1.51984 19.6522 1.60761 19.5002L11.1339 3.00017C11.41 2.52187 12.0216 2.358 12.4999 2.63414C12.6519 2.72191 12.7782 2.84815 12.8659 3.00017ZM10.9999 16.0002V18.0002H12.9999V16.0002H10.9999ZM10.9999 9.00017V14.0002H12.9999V9.00017H10.9999Z"
         id="path1"

src/components/CodeViewGridItem.tsx

@@ -16,10 +16,11 @@ interface CodeViewGridItemProps {
 const CodeViewGridItem = ({ gesture }: CodeViewGridItemProps) => {
   const model = useStore((s) => s.model);
   const gestures = useStore((s) => s.gestures);
+  const dataWindow = useStore((s) => s.dataWindow);
   const project = useMemo(
     // Project name is left empty as it is not used or displayed.
-    () => generateProject("", { data: gestures }, model, gesture),
-    [gesture, gestures, model]
+    () => generateProject("", { data: gestures }, model, dataWindow, gesture),
+    [dataWindow, gesture, gestures, model]
   );
   const width = useMemo(
     () => `${120 + gesture.name.length * 5}px`,

src/components/DefaultPageLayout.tsx

@@ -7,7 +7,6 @@ import {
   IconButton,
   MenuDivider,
   MenuItem,
-  useToast,
   VStack,
 } from "@chakra-ui/react";
 import { ReactNode, useCallback, useEffect } from "react";
@@ -24,7 +23,7 @@ import { useProject } from "../hooks/project-hooks";
 import { SaveStep, TrainModelDialogStage } from "../model";
 import Tour from "../pages/Tour";
 import { useStore } from "../store";
-import { createDataSamplesPageUrl, createHomePageUrl } from "../urls";
+import { createHomePageUrl } from "../urls";
 import ActionBar from "./ActionBar";
 import AppLogo from "./AppLogo";
 import ConnectionDialogs from "./ConnectionFlowDialogs";
@@ -54,7 +53,6 @@ const DefaultPageLayout = ({
   showPageTitle = false,
 }: DefaultPageLayoutProps) => {
   const intl = useIntl();
-  const navigate = useNavigate();
   const isEditorOpen = useStore((s) => s.isEditorOpen);
   const isTourClosed = useStore((s) => s.tourState === undefined);
   const isTrainDialogClosed = useStore(
@@ -63,8 +61,6 @@ const DefaultPageLayout = ({
   const { stage } = useConnectionStage();
   const isConnectionDialogClosed = stage.flowStep === ConnectionFlowStep.None;
   const isSaveDialogClosed = useStore((s) => s.save.step === SaveStep.None);
-
-  const toast = useToast();
   const { appNameFull } = useDeployment();
 
   useEffect(() => {
@@ -73,26 +69,6 @@ const DefaultPageLayout = ({
       : appNameFull;
   }, [appNameFull, intl, titleId]);
 
-  useEffect(() => {
-    return useStore.subscribe(
-      (
-        { projectLoadTimestamp },
-        { projectLoadTimestamp: prevProjectLoadTimestamp }
-      ) => {
-        if (projectLoadTimestamp > prevProjectLoadTimestamp) {
-          // Side effects of loading a project, which MakeCode notifies us of.
-          navigate(createDataSamplesPageUrl());
-          toast({
-            position: "top",
-            duration: 5_000,
-            title: intl.formatMessage({ id: "project-loaded" }),
-            status: "info",
-          });
-        }
-      }
-    );
-  }, [intl, navigate, toast]);
-
   return (
     <>
       {/* Suppress dialogs to prevent overlapping dialogs */}

src/components/LiveGraph.tsx

@@ -10,7 +10,6 @@ import { RiArrowDropLeftFill } from "react-icons/ri";
 import React from "react";
 import { LabelConfig, getUpdatedLabelConfig } from "../live-graph-label-config";
 import { useStore } from "../store";
-import { mlSettings } from "../mlConfig";
 
 const initialLabelConfigs: LabelConfig[] = [
   { label: "x", arrowHeight: 0, labelHeight: 0, color: "#f9808e", id: 0 },
@@ -89,6 +88,7 @@ const LiveGraph = () => {
   // Draw on graph to display that users are recording
   // Ideally we'd do this without timing the recording again!
   const [isTimingRecording, setIsTimingRecording] = useState<boolean>(false);
+  const dataWindow = useStore((s) => s.dataWindow);
   const isRecording = useStore((s) => s.isRecording);
   useEffect(() => {
     if (isRecording && !isTimingRecording) {
@@ -106,9 +106,9 @@ const LiveGraph = () => {
         recordLines.append(now - 1, 2.3, false);
         recordLines.append(now, -2, false);
         setIsTimingRecording(false);
-      }, mlSettings.duration);
+      }, dataWindow.duration);
     }
-  }, [isTimingRecording, recordLines, isRecording]);
+  }, [isTimingRecording, recordLines, isRecording, dataWindow.duration]);
 
   const [labelConfigs, setLabelConfigs] =
     useState<LabelConfig[]>(initialLabelConfigs);

src/components/RecordingDialog.tsx

@@ -18,7 +18,6 @@ import { TimedXYZ } from "../buffered-data";
 import { useBufferedData } from "../buffered-data-hooks";
 import { GestureData, XYZData } from "../model";
 import { useStore } from "../store";
-import { mlSettings } from "../mlConfig";
 
 interface CountdownStage {
   value: string | number;
@@ -231,13 +230,14 @@ interface RecordingDataSource {
 
 const useRecordingDataSource = (): RecordingDataSource => {
   const ref = useRef<InProgressRecording | undefined>();
+  const dataWindow = useStore((s) => s.dataWindow);
   const bufferedData = useBufferedData();
   useEffect(() => {
     const listener = (sample: TimedXYZ) => {
       if (ref.current) {
         const percentage =
           ((sample.timestamp - ref.current.startTimeMillis) /
-            mlSettings.duration) *
+            dataWindow.duration) *
           100;
         ref.current.onProgress(percentage);
       }
@@ -246,7 +246,7 @@ const useRecordingDataSource = (): RecordingDataSource => {
     return () => {
       bufferedData.removeListener(listener);
     };
-  }, [bufferedData]);
+  }, [bufferedData, dataWindow.duration]);
 
   return useMemo(
     () => ({
@@ -257,10 +257,10 @@ const useRecordingDataSource = (): RecordingDataSource => {
           if (ref.current) {
             const data = bufferedData.getSamples(
               ref.current.startTimeMillis,
-              ref.current.startTimeMillis + mlSettings.duration
+              ref.current.startTimeMillis + dataWindow.duration
             );
             const sampleCount = data.x.length;
-            if (sampleCount < mlSettings.minSamples) {
+            if (sampleCount < dataWindow.minSamples) {
               ref.current.onError();
               ref.current = undefined;
             } else {
@@ -269,7 +269,7 @@ const useRecordingDataSource = (): RecordingDataSource => {
               ref.current = undefined;
             }
           }
-        }, mlSettings.duration);
+        }, dataWindow.duration);
 
         ref.current = {
           startTimeMillis: Date.now(),
@@ -281,7 +281,7 @@ const useRecordingDataSource = (): RecordingDataSource => {
         ref.current = undefined;
       },
     }),
-    [bufferedData]
+    [bufferedData, dataWindow.duration, dataWindow.minSamples]
   );
 };
 

src/components/RecordingFingerprint.tsx

@@ -4,6 +4,7 @@ import { applyFilters } from "../ml";
 import { XYZData } from "../model";
 import { calculateGradientColor } from "../utils/gradient-calculator";
 import ClickableTooltip from "./ClickableTooltip";
+import { useStore } from "../store";
 
 interface RecordingFingerprintProps extends BoxProps {
   data: XYZData;
@@ -15,7 +16,8 @@ const RecordingFingerprint = ({
   gestureName,
   ...rest
 }: RecordingFingerprintProps) => {
-  const dataFeatures = applyFilters(data, { normalize: true });
+  const dataWindow = useStore((s) => s.dataWindow);
+  const dataFeatures = applyFilters(data, dataWindow, { normalize: true });
 
   return (
     <Grid

src/hooks/ml-hooks.tsx

@@ -21,6 +21,7 @@ export const usePrediction = () => {
   const [prediction, setPrediction] = useState<PredictionResult | undefined>();
   const gestureData = useStore((s) => s.gestures);
   const model = useStore((s) => s.model);
+  const dataWindow = useStore((s) => s.dataWindow);
 
   // Use a ref to prevent restarting the effect every time thesholds change.
   // We only use the ref's value during the setInterval callback not render.
@@ -32,14 +33,14 @@ export const usePrediction = () => {
       return;
     }
     const runPrediction = async () => {
-      const startTime = Date.now() - mlSettings.duration;
+      const startTime = Date.now() - dataWindow.duration;
       const input = {
         model,
         data: buffer.getSamples(startTime),
         classificationIds: gestureDataRef.current.map((g) => g.ID),
       };
-      if (input.data.x.length > mlSettings.minSamples) {
-        const result = await predict(input);
+      if (input.data.x.length > dataWindow.minSamples) {
+        const result = await predict(input, dataWindow);
         if (result.error) {
           logging.error(result.detail);
         } else {
@@ -63,7 +64,7 @@ export const usePrediction = () => {
       setPrediction(undefined);
       clearInterval(interval);
     };
-  }, [connection, logging, connectStatus, buffer, model]);
+  }, [connection, logging, connectStatus, buffer, model, dataWindow]);
 
   return prediction;
 };