Widget Image Segmentation (#378)

* [WIP] Image Segmentation Widget

* Add css animations & refactoring ImageSegWidget

* Lint/formamt

* Update implemnetation

* Refactor

* Turn off canvas animation

* Turn off widget on Safari

* Chore

* Make dev change for demo

* Add demo img

* Bring back canvas animation

* Detect safari specifically

* Revert "Detect safari specifically"

This reverts commit 9e9091b.

* Chore

* Performant animation

* Rename files

* Give feedback to safari users

* polyfillCreateImageBitmap for safari

* Rm unnecessary lines

* New img-seg mask output shape

* Refactor

* Call api

* Rm dev changes

* Disable file selector when loading

* Add resize handler (FireFox issue)

* Revert dev change

* Fix snake_case -> cameCase

* Rename clip -> clamp

* Add EOF tailwind.css

* Rm setSlice method

* Make addOutputColor functional

* Create consts file

* Obj det use consts

* Update consts file

* Rm unnecessary parts of polufill

* Rm rgb values from colors const

* css drag n drop

* Chore img class

* Rename `modelId` -> `id`

* Apply ObjDet updates

* Update WidgetOutputchart css

* Add examples picker for the img sg

* Fix dropzone pointer-events-none

* chore

* Better canvas interactivity

* Prevent drop on loading doprozone

* Disable examples picker when loading

* Activate outputjson

Author: Mishig Davaadorj

widgets/src/lib/InferenceWidget/InferenceWidget.svelte

@@ -10,6 +10,7 @@
 	import FeatureExtractionWidget from "./widgets/FeatureExtractionWidget/FeatureExtractionWidget.svelte";
 	import FillMaskWidget from "./widgets/FillMaskWidget/FillMaskWidget.svelte";
 	import ImageClassificationWidget from "./widgets/ImageClassificationWidget/ImageClassificationWidget.svelte";
+	import ImageSegmentationWidget from "./widgets/ImageSegmentationWidget/ImageSegmentationWidget.svelte";
 	import ObjectDetectionWidget from "./widgets/ObjectDetectionWidget/ObjectDetectionWidget.svelte";
 	import QuestionAnsweringWidget from "./widgets/QuestionAnsweringWidget/QuestionAnsweringWidget.svelte";
 	import SentenceSimilarityWidget from "./widgets/SentenceSimilarityWidget/SentenceSimilarityWidget.svelte";
@@ -44,6 +45,7 @@
 		"feature-extraction": FeatureExtractionWidget,
 		"fill-mask": FillMaskWidget,
 		"image-classification": ImageClassificationWidget,
+		"image-segmentation": ImageSegmentationWidget,
 		"object-detection": ObjectDetectionWidget,
 		"question-answering": QuestionAnsweringWidget,
 		"sentence-similarity": SentenceSimilarityWidget,

widgets/src/lib/InferenceWidget/shared/ViewUtils.ts

@@ -8,6 +8,13 @@ const ESCAPED = {
 	">": ">",
 };
 
+/**
+ *  Returns a function that clamps input value to range [min <= x <= max].
+ */
+ export function clamp(x: number, min: number, max: number): number {
+	return Math.max(min, Math.min(x, max));
+}
+
 /**
  * HTML escapes the passed string
  */
@@ -120,9 +127,9 @@ export function convertTableToData(table: (string | number)[][]): TableData {
 }
 
 /*
-	* Converts data from {Header0: [ColumnVal0, ...], Header1: [Column1Val0, ...], Header2: [Column2Val0, ...]}
-	* to [[Header0, Header1, Header2], [Column0Val0, Column1Val0, Column2Val0], ...]
-	*/
+* Converts data from {Header0: [ColumnVal0, ...], Header1: [Column1Val0, ...], Header2: [Column2Val0, ...]}
+* to [[Header0, Header1, Header2], [Column0Val0, Column1Val0, Column2Val0], ...]
+*/
 export function convertDataToTable(data: TableData): (string | number)[][] {
 	const dataArray = Object.entries(data); // [header, cell[]][]
 	const nbCols = dataArray.length;
@@ -136,11 +143,22 @@ export function convertDataToTable(data: TableData): (string | number)[][] {
 		);
 }
 
+/*
+* Converts hex string to rgb array (i.e. [r,g,b])
+* original from https://stackoverflow.com/a/39077686/6558628
+*/
+export function hexToRgb(hex: string): number[]{
+	return hex.replace(/^#?([a-f\d])([a-f\d])([a-f\d])$/i
+			   ,(_, r, g, b) => '#' + r + r + g + g + b + b)
+	  .substring(1).match(/.{2}/g)
+	  .map(x => parseInt(x, 16))
+}
+
 /*
 * For Tailwind:
 bg-blue-100 border-blue-100 dark:bg-blue-800 dark:border-blue-800
 bg-green-100 border-green-100 dark:bg-green-800 dark:border-green-800
 bg-yellow-100 border-yellow-100 dark:bg-yellow-800 dark:border-yellow-800
 bg-purple-100 border-purple-100 dark:bg-purple-800 dark:border-purple-800
 bg-red-100 border-red-100 dark:bg-red-800 dark:border-red-800
-*/
+*/

widgets/src/lib/InferenceWidget/shared/WidgetDropzone/WidgetDropzone.svelte

@@ -23,7 +23,7 @@
 	async function onDrop(e: DragEvent) {
 		isDragging = false;
 		const itemList = e.dataTransfer?.items;
-		if (!itemList) {
+		if (!itemList || isLoading) {
 			return;
 		}
 		const items: DataTransferItem[] = [];
@@ -56,6 +56,7 @@
 	{accept}
 	bind:this={fileInput}
 	on:change={onChange}
+	disabled={isLoading}
 	style="display: none;"
 	type="file"
 />
@@ -78,7 +79,9 @@
 	{#if !imgSrc}
 		<span class="pointer-events-none text-sm">{label}</span>
 	{:else}
-		<slot />
+		<div class={isDragging && "pointer-events-none"}>
+			<slot />
+		</div>
 	{/if}
 	{#if isLoading}
 		<div

widgets/src/lib/InferenceWidget/shared/WidgetFileInput/WidgetFileInput.svelte

@@ -6,7 +6,7 @@
 	export let classNames = "";
 	export let isLoading = false;
 	export let label = "Browse for file";
-	export let onSelectFile: (file: File) => void;
+	export let onSelectFile: (file: File | Blob) => void;
 
 	let fileInput: HTMLInputElement;
 	let isDragging = false;

widgets/src/lib/InferenceWidget/shared/WidgetInputSamples/WidgetInputSamples.svelte

@@ -1,6 +1,7 @@
 <script>
 	import { slide } from "svelte/transition";
 
+	export let isLoading = false;
 	export let inputSamples: Record<string, any>[];
 	export let applyInputSample: (sample: Record<string, any>) => void;
 	export let previewInputSample: (sample: Record<string, any>) => void;
@@ -52,7 +53,10 @@
 
 <svelte:window on:click={onClick} />
 
-<div class="relative z-10 ml-2" bind:this={containerEl}>
+<div
+	class="relative z-10 ml-2 {isLoading && 'pointer-events-none opacity-50'}"
+	bind:this={containerEl}
+>
 	<div
 		class="no-hover:hidden inline-flex justify-between w-32 lg:w-44 rounded-md border border-gray-100 px-4 py-1"
 		on:click={toggleOptionsVisibility}

widgets/src/lib/InferenceWidget/shared/WidgetOutputChart/WidgetOutputChart.svelte

@@ -24,11 +24,11 @@ from-lime-400 to-lime-200 dark:from-lime-400 dark:to-lime-600
 </script>
 
 {#if output.length}
-	<div class="space-y-4 {classNames}">
+	<div>
 		{#each output as { label, score, color="purple" }, index}
 			<div
 				class="flex items-start justify-between font-mono text-xs animate__animated animate__fadeIn leading-none 
-					transition duration-200 ease-in-out
+					transition duration-200 ease-in-out {classNames}
 					{highlightIndex !== -1 &&
 					highlightIndex !== index &&
 					'opacity-30 filter grayscale'}

widgets/src/lib/InferenceWidget/shared/WidgetWrapper/WidgetWrapper.svelte

@@ -13,6 +13,7 @@
 	export let apiUrl: string;
 	export let computeTime: string;
 	export let error: string;
+	export let isLoading = false;
 	export let model: WidgetProps["model"];
 	export let modelLoading = {
 		isLoading: false,
@@ -64,6 +65,7 @@
 			{#if inputSamples.length}
 				<!-- Show samples selector when there are more than one sample -->
 				<WidgetInputSamples
+					{isLoading}
 					{inputSamples}
 					{applyInputSample}
 					{previewInputSample}

widgets/src/lib/InferenceWidget/shared/consts.ts

@@ -0,0 +1,37 @@
+/*
+ * Color palette for obj-det & img-seg widgets
+ */
+export const COLORS = [
+	{
+		"color": "red",
+		"hex": "#f87171",
+	},
+	{
+		"color": "green",
+		"hex": "#4ade80",
+	},
+	{
+		"color": "yellow",
+		"hex": "#facc15",
+	},
+	{
+		"color": "blue",
+		"hex": "#60a5fa",
+	},
+	{
+		"color": "orange",
+		"hex": "#fb923c",
+	},
+	{
+		"color": "purple",
+		"hex": "#c084fc",
+	},
+	{
+		"color": "cyan",
+		"hex": "#22d3ee",
+	},
+	{
+		"color": "lime",
+		"hex": "#a3e635",
+	}
+] as const;

widgets/src/lib/InferenceWidget/shared/types.ts

@@ -29,3 +29,11 @@ export type DetectedObject = {
 	score: number;
 	color?: string;
 }
+export interface ImageSegment {
+	label: string;
+	score: number;
+	mask: string;
+	color?: string;
+	imgData?: ImageData;
+	bitmap?: ImageBitmap;
+};

widgets/src/lib/InferenceWidget/widgets/AudioClassificationWidget/AudioClassificationWidget.svelte

@@ -146,6 +146,7 @@
 	{applyInputSample}
 	{computeTime}
 	{error}
+	{isLoading}
 	{model}
 	{modelLoading}
 	{noTitle}
@@ -187,6 +188,6 @@
 		</form>
 	</svelte:fragment>
 	<svelte:fragment slot="bottom">
-		<WidgetOutputChart classNames="mt-4" {output} />
+		<WidgetOutputChart classNames="pt-4" {output} />
 	</svelte:fragment>
 </WidgetWrapper>