feat: Merge pull request #308 from UniversalDataTool/feat/pose-estimation

fixes #285, Adds pose/keypoints interface

Author: seveibar

package.json

@@ -110,7 +110,7 @@
     "react-hotkeys": "^2.0.0",
     "react-i18next": "^11.4.0",
     "react-icons": "^3.9.0",
-    "react-image-annotate": "^1.6.1",
+    "react-image-annotate": "^1.7.0",
     "react-material-workspace-layout": "^0.1.6",
     "react-scripts": "^3.4.1",
     "react-select": "^3.0.8",

src/components/ConfigureAudioTranscription/index.jsx renamed to src/components/ConfigureAudioTranscription/index.js

@@ -0,0 +1,15 @@
+import React from "react"
+
+export default ({ iface, onChange }) => {
+  return (
+    <div>
+      We'd like to put a little interface to edit poses or landmarks here, but
+      we need help! Check out this{" "}
+      <a href="https://github.com/UniversalDataTool/universal-data-tool/issues/307">
+        Github Issue
+      </a>{" "}
+      to learn more! For now just turn to the JSON tab to edit the landmarks
+      manually!
+    </div>
+  )
+}

src/components/ConfigureImageLandmarkAnnotation/index.js

@@ -14,6 +14,7 @@ import ConfigureDataEntry from "../ConfigureDataEntry"
 import ConfigureComposite from "../ConfigureComposite"
 import Configure3D from "../Configure3D"
 import ConfigureVideoSegmentation from "../ConfigureVideoSegmentation"
+import ConfigureImageLandmarkAnnotation from "../ConfigureImageLandmarkAnnotation"
 
 const Container = styled("div")({
   padding: 24,
@@ -72,6 +73,9 @@ export const ConfigureInterface = ({
       {iface.type === "3d_bounding_box" && (
         <Configure3D iface={iface} onChange={onChange} />
       )}
+      {iface.type === "image_landmark_annotation" && (
+        <ConfigureImageLandmarkAnnotation iface={iface} onChange={onChange} />
+      )}
     </Container>
   )
 }

src/components/ConfigureInterface/index.js

@@ -0,0 +1,107 @@
+// @flow
+
+import React, { useMemo } from "react"
+import Annotator from "react-image-annotate"
+import { styled } from "@material-ui/core/styles"
+import useEventCallback from "use-event-callback"
+import {
+  convertFromRIARegionFmt,
+  convertToRIAImageFmt,
+} from "../../utils/ria-format.js"
+import useClobberedState from "../../utils/use-clobbered-state"
+
+const Container = styled("div")({
+  "& .fullscreen": { height: "100%" },
+})
+
+const [emptyObj, emptyArr] = [{}, []]
+
+const posingTools = ["create-keypoints"]
+
+export default ({
+  sampleIndex: globalSampleIndex,
+  interface: iface,
+  sampleIndex,
+  samples = emptyArr,
+  containerProps = emptyObj,
+  onSaveTaskOutputItem,
+  onModifySample,
+}) => {
+  const [selectedIndex, setSelectedIndex] = useClobberedState(
+    globalSampleIndex,
+    0
+  )
+
+  const saveCurrentIndexAnnotation = useEventCallback((output) => {
+    const img = output.images[0]
+    const annotation = (img.regions || []).map(convertFromRIARegionFmt)
+    if (onModifySample) {
+      onModifySample(selectedIndex, { annotation })
+    } else {
+      onSaveTaskOutputItem(selectedIndex, annotation)
+    }
+  })
+
+  const onExit = useEventCallback((output, nextAction) => {
+    saveCurrentIndexAnnotation(output)
+    if (containerProps.onExit) containerProps.onExit(nextAction)
+  })
+  const onNextImage = useEventCallback((output) => {
+    if (selectedIndex + 1 >= samples.length) {
+      onExit(output, "go-to-next")
+    } else {
+      saveCurrentIndexAnnotation(output)
+      if (setSelectedIndex) {
+        setSelectedIndex(selectedIndex + 1)
+      } else {
+        onExit(output, "go-to-next")
+      }
+    }
+  })
+  const onPrevImage = useEventCallback((output) => {
+    if (selectedIndex - 1 < 0) {
+      onExit(output, "go-to-previous")
+    } else {
+      saveCurrentIndexAnnotation(output)
+      if (setSelectedIndex) {
+        setSelectedIndex(selectedIndex - 1)
+      } else {
+        onExit(output, "go-to-previous")
+      }
+    }
+  })
+
+  const singleImageList = useMemo(() => {
+    return [
+      convertToRIAImageFmt({
+        title: containerProps.datasetName || `Sample ${selectedIndex}`,
+        taskDatum: samples[selectedIndex],
+        output: samples[selectedIndex].annotation,
+        selectedIndex,
+      }),
+    ]
+    // eslint-disable-next-line
+  }, [selectedIndex, containerProps.datasetName])
+
+  return (
+    <Container
+      style={{
+        height: containerProps.height || "calc(100% - 70px)",
+        minHeight: 600,
+        width: "100%",
+      }}
+    >
+      <Annotator
+        key={globalSampleIndex}
+        keypointDefinitions={iface.keypointDefinitions}
+        selectedImage={0}
+        taskDescription={iface.description}
+        onNextImage={onNextImage}
+        onPrevImage={onPrevImage}
+        enabledTools={posingTools}
+        images={singleImageList}
+        onExit={onExit}
+      />
+    </Container>
+  )
+}

src/components/ImageLandmarkAnnotation/index.js

@@ -0,0 +1,78 @@
+// @flow
+
+import React from "react"
+
+import { storiesOf } from "@storybook/react"
+import { action } from "@storybook/addon-actions"
+
+import ImageLandmarkAnnotation from "./"
+
+storiesOf("ImageLandmarkAnnotation", module).add("Basic", () => (
+  <ImageLandmarkAnnotation
+    onModifySample={action("onModifySample")}
+    {...{
+      interface: {
+        type: "image_landmark_annotation",
+
+        // Different configurations of poses / keypoints
+        keypointDefinitions: {
+          // This is a pose definition for the upper half of a human
+          human: {
+            // Each landmark is a point on the pose.
+            landmarks: {
+              head: {
+                label: "Head",
+                color: "#f00",
+                // The default position is the position relative to the cursor
+                // in [image_width%, image_height%] to place this point at when
+                // a pose is created
+                defaultPosition: [0, -0.05],
+              },
+              sternum: {
+                label: "Torso",
+                color: "#0f0",
+                defaultPosition: [0, 0],
+              },
+              leftElbow: {
+                label: "Left Elbow",
+                color: "#00f",
+                defaultPosition: [-0.05, 0],
+              },
+              rightElbow: {
+                label: "Right Elbow",
+                color: "#00f",
+                defaultPosition: [0.05, 0],
+              },
+            },
+
+            // The connections will determine what lines are drawn between points, they
+            // are only aesthetic
+            connections: [
+              ["head", "sternum"],
+              ["sternum", "leftElbow"],
+              ["sternum", "rightElbow"],
+            ],
+          },
+        },
+      },
+      samples: [
+        {
+          // URL pointing to image
+          imageUrl:
+            "https://media.gettyimages.com/photos/dog-and-cat-picture-id151350785",
+          annotation: {
+            regionType: "keypoints",
+            keypointsDefinitionId: "human",
+            points: {
+              head: { x: 0.54, y: 0.2 },
+              sternum: { x: 0.57, y: 0.3 },
+              // when "obscured" is set to true, it means something is blocking or obstructing the visibility of this keypoint in the image
+              leftElbow: { x: 0.4, y: 0.39, obscured: true },
+              rightElbow: { x: 0.7, y: 0.32 },
+            },
+          },
+        },
+      ],
+    }}
+  />
+))

src/components/ImageLandmarkAnnotation/index.story.js

@@ -0,0 +1,22 @@
+// @flow
+
+import React, { useState } from "react"
+
+import { storiesOf } from "@storybook/react"
+import { action } from "@storybook/addon-actions"
+
+import SetupPage from "./"
+
+storiesOf("SetupPage", module).add("Empty Dataset", () => {
+  const [dataset, setDataset] = useState({
+    interface: {},
+  })
+
+  return (
+    <SetupPage
+      dataset={dataset}
+      onChange={setDataset}
+      onClearLabelData={action("onClearLabelData")}
+    />
+  )
+})

src/components/SetupPage/index.story.js

@@ -11,6 +11,7 @@ import ThreeDRotation from "@material-ui/icons/ThreeDRotation"
 import OndemandVideoIcon from "@material-ui/icons/OndemandVideo"
 import ContactSupport from "@material-ui/icons/ContactSupport"
 import LowPriority from "@material-ui/icons/LowPriority"
+import AccessibilityNewIcon from "@material-ui/icons/AccessibilityNew"
 
 export const templates = [
   {
@@ -291,6 +292,136 @@ export const templates = [
       ],
     },
   },
+  {
+    name: "Image Landmark Annotation",
+    Icon: AccessibilityNewIcon,
+    dataset: {
+      interface: {
+        type: "image_landmark_annotation",
+        keypointDefinitions: {
+          human: {
+            landmarks: {
+              nose: {
+                label: "Nose",
+                color: "#f00",
+                defaultPosition: [0, 0],
+              },
+              leftEye: {
+                label: "Left Eye",
+                color: "#00f",
+                defaultPosition: [-0.05, -0.05],
+              },
+              leftEar: {
+                label: "Left Ear",
+                color: "#0ff",
+                defaultPosition: [-0.1, -0.05],
+              },
+              rightEye: {
+                label: "Left Eye",
+                color: "#f0f",
+                defaultPosition: [0.05, -0.05],
+              },
+              rightEar: {
+                label: "Left Ear",
+                color: "#00f",
+                defaultPosition: [0.1, -0.05],
+              },
+              sternum: {
+                label: "Sternum",
+                color: "#0f0",
+                defaultPosition: [0, 0.1],
+              },
+              leftShoulder: {
+                label: "Left Shoulder",
+                color: "#0ff",
+                defaultPosition: [-0.2, 0.1],
+              },
+              rightShoulder: {
+                label: "Right Shoulder",
+                color: "#00f",
+                defaultPosition: [0.2, 0.1],
+              },
+              leftElbow: {
+                label: "Left Elbow",
+                color: "#0f0",
+                defaultPosition: [-0.2, 0.2],
+              },
+              rightElbow: {
+                label: "Right Elbow",
+                color: "#f00",
+                defaultPosition: [0.2, 0.2],
+              },
+              leftHand: {
+                label: "Left Hand",
+                color: "#00f",
+                defaultPosition: [-0.2, 0.3],
+              },
+              rightHand: {
+                label: "Right Hand",
+                color: "#f0f",
+                defaultPosition: [0.2, 0.3],
+              },
+              leftThigh: {
+                label: "Left Thigh",
+                color: "#f00",
+                defaultPosition: [-0.1, 0.35],
+              },
+              rightThigh: {
+                label: "Right Thigh",
+                color: "#0ff",
+                defaultPosition: [0.1, 0.35],
+              },
+              leftKnee: {
+                label: "Left Thigh",
+                color: "#ff0",
+                defaultPosition: [-0.15, 0.45],
+              },
+              rightKnee: {
+                label: "Right Thigh",
+                color: "#0f0",
+                defaultPosition: [0.15, 0.45],
+              },
+              leftFoot: {
+                label: "Left Foot",
+                color: "#00f",
+                defaultPosition: [-0.15, 0.55],
+              },
+              rightFoot: {
+                label: "Right Foot",
+                color: "#f00",
+                defaultPosition: [0.15, 0.55],
+              },
+            },
+            connections: [
+              ["sternum", "nose"],
+              ["nose", "leftEye"],
+              ["leftEye", "leftEar"],
+              ["nose", "rightEye"],
+              ["rightEye", "rightEar"],
+              ["sternum", "leftShoulder"],
+              ["leftShoulder", "leftElbow"],
+              ["leftElbow", "leftHand"],
+              ["sternum", "leftThigh"],
+              ["leftThigh", "leftKnee"],
+              ["leftKnee", "leftFoot"],
+              ["sternum", "rightShoulder"],
+              ["rightShoulder", "rightElbow"],
+              ["rightElbow", "rightHand"],
+              ["sternum", "rightThigh"],
+              ["rightThigh", "rightKnee"],
+              ["rightKnee", "rightFoot"],
+            ],
+          },
+        },
+      },
+      samples: [
+        {
+          imageUrl:
+            "https://media.istockphoto.com/photos/businesswoman-picture-id918002786",
+        },
+      ],
+    },
+  },
   {
     name: "3D Bounding Box",
     Icon: ThreeDRotation,