Implement own version of fingerprint visualization from TFlow

r59q · r59q · commit e55d64412c86 · 2025-07-08T21:52:28.000+02:00
diff --git a/features.json b/features.json
@@ -6,5 +6,6 @@
   "liveGraphInputValues": true,
   "recordingScrubberValues": true,
   "modelValidation": true,
-  "modelSettings": true
+  "modelSettings": true,
+  "fingerprint": true
 }
diff --git a/src/__tests__/archtest/default-build-config.test.ts b/src/__tests__/archtest/default-build-config.test.ts
@@ -31,4 +31,9 @@ describe('Default build config test', () => {
     expect(windiContent.includes("secondary: '#2CCAC0'"), message).toBeFalsy();
     expect(features.title, message).toBe('Learning tool');
   });
+
+  test("deleteme pls", () => {
+    // TODO: Remember to add feature flag for fingerprint!
+    expect(true).toBeFalsy();
+  });
 });
diff --git a/src/__viteBuildVariants__/ml-machine-simple/features.json b/src/__viteBuildVariants__/ml-machine-simple/features.json
@@ -6,5 +6,6 @@
   "liveGraphInputValues": false,
   "recordingScrubberValues": false,
   "modelValidation": false,
-  "modelSettings": false
+  "modelSettings": false,
+  "fingerprint": false
 }
diff --git a/src/__viteBuildVariants__/ml-machine/features.json b/src/__viteBuildVariants__/ml-machine/features.json
@@ -6,5 +6,6 @@
   "liveGraphInputValues": true,
   "recordingScrubberValues": true,
   "modelValidation": true,
-  "modelSettings": true
+  "modelSettings": true,
+  "fingerprint": true
 }
diff --git a/src/__viteBuildVariants__/unbranded/features.json b/src/__viteBuildVariants__/unbranded/features.json
@@ -6,5 +6,6 @@
   "liveGraphInputValues": true,
   "recordingScrubberValues": true,
   "modelValidation": true,
-  "modelSettings": true
+  "modelSettings": true,
+  "fingerprint": true
 }
diff --git a/src/assets/messages/ui.da.json b/src/assets/messages/ui.da.json
@@ -50,6 +50,7 @@
   "content.data.noData.exampleName.shake": "Ryste",
   "content.data.noData.exampleName.still": "Stille",
   "content.data.noData.exampleName.circle": "Cirkel",
+  "content.data.tooltip.remove": "Fjern",
   "content.trainer.failure.header": "Træning mislykkedes",
   "content.trainer.failure.body": "Træningen resulterede ikke i en brugbar model. Grunden til dette ligger sandsynligvis i dataet. Hvis dataet i forskellige klasser minder for meget om hinanden, kan dette resultere i nogle forskellige problemer i træningsprocessen, der ikke gør det muligt at træne modellen ordentligt.",
   "content.trainer.failure.todo": "Gå tilbage til datasiden og ændr i din data.",
diff --git a/src/assets/messages/ui.de.json b/src/assets/messages/ui.de.json
@@ -50,6 +50,7 @@
   "content.data.noData.exampleName.shake": "Schütteln",
   "content.data.noData.exampleName.still": "Still",
   "content.data.noData.exampleName.circle": "Kreis",
+  "content.data.tooltip.remove": "Entfernen",
   "content.trainer.failure.header": "Training fehlgeschlagen",
   "content.trainer.failure.body": "Das Training führte nicht zu einem brauchbaren Modell. Der Grund dafür sind höchstwahrscheinlich die für das Training verwendeten Daten. Wenn die Daten für verschiedene Klassen zu ähnlich sind, kann dies zu Problemen im Trainingsprozess führen.",
   "content.trainer.failure.todo": "Gehe zur Datenseite zurück und ändere die Daten.",
diff --git a/src/assets/messages/ui.en.json b/src/assets/messages/ui.en.json
@@ -50,6 +50,7 @@
   "content.data.noData.exampleName.shake": "Shake",
   "content.data.noData.exampleName.still": "Still",
   "content.data.noData.exampleName.circle": "Circle",
+  "content.data.tooltip.remove": "Remove",
   "content.trainer.failure.header": "Training Failed",
   "content.trainer.failure.body": "The training did not result in a usable model. The reason for this is most likely the data used for training. If the data for different classes are too similar, this can result in issues in the training process.",
   "content.trainer.failure.todo": "Return to the data page and change your data.",
diff --git a/src/components/features/bottom/BottomPanel.svelte b/src/components/features/bottom/BottomPanel.svelte
@@ -83,7 +83,7 @@
         class="absolute right-0 cursor-pointer w-45 hover:bg-secondary hover:bg-opacity-10 transition"
         on:click={() => (isLive3DOpen = true)}>
         <div class="flex flex-row">
-          <div class="absolute">
+          <div class="absolute h-full">
             <LiveDataFingerprint gestureName="Live" />
           </div>
           <View3DLive width={160} height={160} freeze={isLive3DOpen} />
diff --git a/src/components/features/bottom/LiveDataFingerprint.svelte b/src/components/features/bottom/LiveDataFingerprint.svelte
@@ -4,27 +4,50 @@
   SPDX-License-Identifier: MIT
  -->
 <script lang="ts">
+  import { onMount } from 'svelte';
+  import { ClassifierInput } from '../../../lib/domain/ClassifierInput';
   import { stores } from '../../../lib/stores/Stores';
+  import StaticConfiguration from '../../../StaticConfiguration';
+  import StandardButton from '../../ui/buttons/StandardButton.svelte';
   import Fingerprint from '../../ui/recording/Fingerprint.svelte';
 
   export let gestureName: string;
   const classifier = stores.getClassifier();
   const filters = classifier.getFilters();
+  const highlightedAxes = stores.getHighlightedAxes();
+  $: liveData = $stores.liveData;
+  let filteredNormalizedInput: null | number[] = null;
 
   $: filtersLabels = $filters.flatMap(filter => {
     const filterName = filter.getName();
     return [`${filterName} - x`, `${filterName} - y`, `${filterName} - z`];
   });
-  $: fingerprint = $classifier.filteredInput.normalized.getValue();
+  // $: fingerprint = $classifier.filteredInput.normalized.getValue();
+  onMount(() => {
+    return liveData?.subscribe(() => {
+      try {
+        if (liveData) {
+          const bufferedData = liveData
+            .getBuffer()
+            .getSeries(
+              StaticConfiguration.pollingPredictionSampleDuration,
+              StaticConfiguration.pollingPredictionSampleSize,
+            );
+          filteredNormalizedInput = ClassifierInput.getInputForAxes(
+            bufferedData.map(e => e.value),
+            $highlightedAxes,
+          ).getNormalizedInput(filters);
+        }
+      } catch (error) {}
+    });
+  });
 </script>
 
-<div class="w-7 overflow-hidden">
-  <div style="margin-left: -8px;margin-bottom: -7px">
+<div class="w-7 h-full">
+  {#if !!filteredNormalizedInput}
     <Fingerprint
-      height={166}
-      width={100}
       filterLabels={filtersLabels}
       title={gestureName}
-      {fingerprint} />
-  </div>
+      fingerprint={filteredNormalizedInput} />
+  {/if}
 </div>
diff --git a/src/components/ui/Tooltip.svelte b/src/components/ui/Tooltip.svelte
@@ -29,7 +29,7 @@
   {#if isHovered && !!title}
     <div
       style="top: {offset.y}px; left: {offset.x}px;"
-      class="absolute p-1 rounded-sm bg-white shadow-md border-1 border-solid">
+      class="absolute p-1 rounded-sm bg-white shadow-md border-1 border-solid z-1">
       {title}
     </div>
   {/if}
diff --git a/src/components/ui/recording/Fingerprint.svelte b/src/components/ui/recording/Fingerprint.svelte
@@ -4,51 +4,96 @@
   SPDX-License-Identifier: MIT
  -->
 <script lang="ts">
-  import * as tfvis from '@tensorflow/tfjs-vis';
-  import { onMount } from 'svelte';
-  import Logger from '../../../lib/utils/Logger';
+    import { stores } from "../../../lib/stores/Stores";
 
   export let fingerprint: number[];
   export let title: string;
   export let filterLabels: string[];
-  export let height: number = 100;
-  export let width: number = 80;
 
-  let surface: undefined | tfvis.Drawable;
+  // Tooltip state
+  let tooltipVisible = false;
+  let tooltipLabel = "";
+  let tooltipValue = 0;
+  let containerElement: HTMLDivElement;
+  let tooltipX = 0;
+  let tooltipXOffset = -80;
+  let tooltipY = 0;
+  let tooltipYOffset = -60;
 
-  // Reactive chart data that updates when props change
-  $: chartData = {
-    values: [fingerprint],
-    xTickLabels: [title],
-    yTickLabels: filterLabels,
-  };
+  // Function to convert value (0-1) to viridis-like color
+  const getViridisColor = (value: number): string => {
+    // Clamp value between 0 and 1
+    const t = Math.max(0, Math.min(1, value));
 
-  const render = async () => {
-    if (!surface) {
-      return;
-    }
-    try {
-      await tfvis.render.heatmap(surface, chartData, {
-        colorMap: 'viridis',
-        height: height,
-        width: width,
-        domain: [0, 1],
-        fontSize: 0,
-      });
-    } catch (error) {
-      Logger.log('Fingerprint failed to render', error);
+    // Viridis color interpolation (simplified)
+    const colors = [
+      [68, 1, 84], // Dark purple
+      [59, 82, 139], // Blue-purple
+      [33, 144, 140], // Teal
+      [94, 201, 98], // Green
+      [253, 231, 37], // Yellow
+    ];
+
+    const scaledT = t * (colors.length - 1);
+    const index = Math.floor(scaledT);
+    const fraction = scaledT - index;
+
+    if (index >= colors.length - 1) {
+      const [r, g, b] = colors[colors.length - 1];
+      return `rgb(${r}, ${g}, ${b})`;
     }
+
+    const [r1, g1, b1] = colors[index];
+    const [r2, g2, b2] = colors[index + 1];
+
+    const r = Math.round(r1 + (r2 - r1) * fraction);
+    const g = Math.round(g1 + (g2 - g1) * fraction);
+    const b = Math.round(b1 + (b2 - b1) * fraction);
+
+    return `rgb(${r}, ${g}, ${b})`;
   };
 
-  onMount(() => {
-    // Initial render
-    render();
-  });
+  // Tooltip handlers
+  const showTooltip = (index: number, value: number) => {
+    tooltipLabel = filterLabels[index] || `Cell ${index}`;
+    tooltipValue = value;
+    
+    // Calculate fixed position relative to the visualization container
+    if (containerElement) {
+      const rect = containerElement.getBoundingClientRect();
+      tooltipX = rect.right + tooltipXOffset; // Position near the right edge of the container
+      tooltipY = rect.top + tooltipYOffset; // Position near the top of the container
+    }
+    
+    tooltipVisible = true;
+  };
 
-  // Reactive statement to rerender when data changes
-  $: if (surface && (fingerprint || filterLabels || title)) {
-    render();
-  }
+  const hideTooltip = () => {
+    tooltipVisible = false;
+  };
 </script>
 
-<div bind:this={surface}></div>
+<div class="flex flex-col font-sans w-full h-full" bind:this={containerElement}>
+  <div class="flex flex-col border border-gray-300 flex-1">
+    {#each fingerprint as value, index}
+        <div
+          class="border-b border-white border-opacity-10 transition-opacity duration-200 hover:opacity-80 cursor-pointer last:border-b-0 flex-1"
+          style="background-color: {getViridisColor(value)};"
+          on:mouseenter={() => showTooltip(index, value)}
+          on:mouseleave={hideTooltip}
+          on:click|stopPropagation
+          role="button"
+          tabindex="0"/>
+    {/each}
+  </div>
+</div>
+
+<!-- Fixed tooltip positioned globally but relative to the visualization -->
+{#if tooltipVisible}
+  <div 
+    class="fixed bg-white shadow-md text-primarytext text-sm px-3 py-2 rounded z-50 pointer-events-none whitespace-nowrap"
+    style="left: {tooltipX}px; top: {tooltipY}px;">
+    <div class="font-semibold">{tooltipLabel}</div>
+    <div>Value: {tooltipValue.toFixed(3)}</div>
+  </div>
+{/if}
diff --git a/src/components/ui/recording/Recording.svelte b/src/components/ui/recording/Recording.svelte
@@ -13,8 +13,9 @@
   import type { RecordingData } from '../../../lib/domain/RecordingData';
   import Tooltip from './../Tooltip.svelte';
   import { serializeRecordingToCsvWithoutGestureName } from '../../../lib/utils/CSVUtils';
-  import Fingerprint from './Fingerprint.svelte';
   import RecordingFingerprint from './RecordingFingerprint.svelte';
+  import { Feature, hasFeature } from '../../../lib/FeatureToggles';
+    import { tr } from '../../../i18n';
 
   // get recording from mother prop
   export let recording: RecordingData;
@@ -57,42 +58,45 @@
     // Clean up the URL object
     URL.revokeObjectURL(url);
   }
+
+  const shouldDisplayFingerprint = enableFingerprint && hasFeature(Feature.FINGERPRINT);
 </script>
 
 <div
   class="h-28 w-50 pr-3 pt-1 relative rounded-md"
-  class:w-40={!enableFingerprint}
-  class:w-50={enableFingerprint}>
+  class:w-40={!shouldDisplayFingerprint}
+  class:w-50={shouldDisplayFingerprint}>
   {#if dotGesture !== undefined}
     <div
       class="absolute px-1 py-0.5 z-3 right-1 top-2"
-      class:right-1={!enableFingerprint}
-      class:right-10={enableFingerprint}>
+      class:right-1={!shouldDisplayFingerprint}
+      class:right-10={shouldDisplayFingerprint}>
       <GestureDot gesture={dotGesture} />
     </div>
   {/if}
   {#if hide}
     <div
       transition:fade
       class="absolute h-26 bg-white"
-      class:w-40={!enableFingerprint}
-      class:w-50={enableFingerprint} />
+      class:w-40={!shouldDisplayFingerprint}
+      class:w-50={shouldDisplayFingerprint} />
   {:else}
     <div
       transition:fade
       class="absolute h-26 bg-white rounded-md"
-      class:w-40={!enableFingerprint}
-      class:w-50={enableFingerprint}>
+      class:w-40={!shouldDisplayFingerprint}
+      class:w-50={shouldDisplayFingerprint}>
       <div class="w-40 h-26">
         <RecordingGraph {recording} />
       </div>
-      {#if enableFingerprint}
-        <div class="absolute right-0 top-0.5 left-40 w-10 overflow-hidden">
+      {#if shouldDisplayFingerprint}
+        <div class="absolute top-0 left-40 h-24.5 w-10 overflow-hidden">
           <RecordingFingerprint {recording} gestureName={$gesture.name} />
         </div>
       {/if}
     </div>
   {/if}
+    <Tooltip title={$tr("content.data.tooltip.remove")} offset={{ x: -26, y: -50 }}>
   <button class="absolute -left-2.8px top-0px outline-none">
     <div class="relative">
       <i class="z-1 absolute fas fa-circle fa-lg text-white" />
@@ -102,16 +106,15 @@
         on:click={deleteClicked} />
     </div>
   </button>
+    </Tooltip>
 
-  <!-- bottom-right button -->
+  <!-- Download Button -->
   {#if downloadable}
-    <Tooltip title="CSV" offset={{ x: 125, y: 125 }}>
+    <Tooltip title="CSV" offset={{ x: 12, y: -50 }}>
       <button
-        class="absolute bottom-4 text-primarytext bg-primary bg-opacity-10 px-2 py-1 text-sm rounded-full shadow-md hover:bg-secondary hover:bg-opacity-30 transition z-1"
-        class:right-10.5={enableFingerprint}
-        class:right-0.5={!enableFingerprint}
+      class="absolute top-0px left-6 text-light-800 hover:text-black transition ease"
         on:click={bottomRightButtonClicked}>
-        <i class="fas fa-download" />
+        <i class="fas fa-download z-1 absolute fa-md" />
       </button>
     </Tooltip>
   {/if}
diff --git a/src/components/ui/recording/RecordingFingerprint.svelte b/src/components/ui/recording/RecordingFingerprint.svelte
@@ -40,8 +40,4 @@
   const fingerprint: number[] = getFilteredInput().getValue();
 </script>
 
-<div class="relative w-full w-11 h-93px overflow-hidden rounded-sm mr-3">
-  <div class="absolute h-full w-full -left-10px overflow-hidden">
-    <Fingerprint filterLabels={filtersLabels} title={gestureName} {fingerprint} />
-  </div>
-</div>
+<Fingerprint filterLabels={filtersLabels} title={gestureName} {fingerprint} />
diff --git a/src/lib/FeatureToggles.ts b/src/lib/FeatureToggles.ts
@@ -16,6 +16,7 @@ export enum Feature {
   RECORDING_SCRUBBER_VALUES = 'recordingScrubberValues',
   MODEL_VALIDATION = 'modelValidation',
   MODEL_SETTINGS = 'modelSettings',
+  FINGERPRINT = 'fingerprint',
 }
 
 export const hasFeature = (feature: Feature): boolean => {

Original file line number	Diff line number	Diff line change
`@@ -6,5 +6,6 @@`
`6`	`6`	`"liveGraphInputValues": true,`
`7`	`7`	`"recordingScrubberValues": true,`
`8`	`8`	`"modelValidation": true,`
`9`		`- "modelSettings": true`
	`9`	`+ "modelSettings": true,`
	`10`	`+ "fingerprint": true`
`10`	`11`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,5 +6,6 @@`
`6`	`6`	`"liveGraphInputValues": false,`
`7`	`7`	`"recordingScrubberValues": false,`
`8`	`8`	`"modelValidation": false,`
`9`		`- "modelSettings": false`
	`9`	`+ "modelSettings": false,`
	`10`	`+ "fingerprint": false`
`10`	`11`	`}`