fix: Merge pull request #202 from UniversalDataTool/feat/full-segmentation

Updates and improvements to full segmentation

Author: seveibar

package-lock.json

@@ -103,7 +103,7 @@
     "react-github-btn": "^1.2.0",
     "react-hotkeys": "^2.0.0",
     "react-icons": "^3.9.0",
-    "react-image-annotate": "^1.2.1",
+    "react-image-annotate": "^1.2.7",
     "react-scripts": "^3.4.1",
     "react-select": "^3.0.8",
     "rfc6902": "^3.0.4",

package.json

@@ -3,6 +3,11 @@ import React, { useMemo } from "react"
 import Survey from "material-survey/components/Survey"
 import { setIn } from "seamless-immutable"
 
+const autoSegmentationOptions = {
+  simple: { type: "simple" },
+  autoseg: { type: "autoseg" },
+}
+
 const form = {
   questions: [
     {
@@ -26,6 +31,12 @@ const form = {
       description: "Any instructions or notes in markdown.",
       type: "text",
     },
+    {
+      name: "autoseg",
+      type: "dropdown",
+      title: "Automatic Segmentation Engine",
+      choices: ["simple", "autoseg"],
+    },
   ],
 }
 
@@ -37,18 +48,31 @@ export default ({ iface, onChange }) => {
         (iface.labels || []).map((a) =>
           typeof a === "string" ? { id: a, description: a } : a
         ) || [],
+      autoseg: iface.autoSegmentationEngine?.type,
     }),
     [iface]
   )
   return (
-    <Survey
-      noActions
-      variant="flat"
-      defaultAnswers={defaultAnswers}
-      onQuestionChange={(questionId, newValue, answers) => {
-        onChange(setIn(iface, [questionId], newValue))
-      }}
-      form={form}
-    />
+    <>
+      <Survey
+        noActions
+        variant="flat"
+        defaultAnswers={defaultAnswers}
+        onQuestionChange={(questionId, newValue, answers) => {
+          if (questionId === "autoseg") {
+            onChange(
+              setIn(
+                iface,
+                ["autoSegmentationEngine"],
+                autoSegmentationOptions[newValue]
+              )
+            )
+          } else {
+            onChange(setIn(iface, [questionId], newValue))
+          }
+        }}
+        form={form}
+      />
+    </>
   )
 }

src/components/ConfigureImagePixelSegmentation/index.js

@@ -10,7 +10,7 @@ import {
 
 const regionTypeToTool = {
   "bounding-box": "create-box",
-  polygon: "create-polygon",
+  polygon: ["create-polygon", "create-expanding-line"],
   point: "create-point",
 }
 
@@ -115,17 +115,20 @@ export default ({
           () =>
             ["select"].concat(
               regionTypesAllowed
-                .map((rt) => regionTypeToTool[rt])
+                .flatMap((rt) => regionTypeToTool[rt])
                 .filter(Boolean)
             ),
           [regionTypesAllowed]
         )
 
   const allowedArea = useMemo(() => {
-    if (!iface.allowedArea) return undefined
-    const { x, y, width: w, height: h } = iface.allowedArea
+    if (!iface.allowedArea && !samples[selectedIndex].allowedArea)
+      return undefined
+    const { x, y, width: w, height: h } =
+      samples[selectedIndex].allowedArea || iface.allowedArea
     return { x, y, w, h }
-  }, [iface.allowedArea])
+    // eslint-disable-next-line
+  }, [iface.allowedArea, samples[selectedIndex].allowedArea])
 
   return (
     <div
@@ -140,9 +143,10 @@ export default ({
         fullImageSegmentationMode={isPixel}
         selectedImage={samples[selectedIndex].imageUrl}
         taskDescription={iface.description}
-        allowedArea={allowedArea}
         showTags={showTags}
         {...labelProps}
+        autoSegmentationOptions={iface.autoSegmentationEngine}
+        allowedArea={allowedArea}
         onNextImage={onNextImage}
         onPrevImage={onPrevImage}
         enabledTools={enabledTools}

src/components/ImageSegmentation/index.js

@@ -0,0 +1,80 @@
+// @flow weak
+
+import React, { useState } from "react"
+import SimpleDialog from "../SimpleDialog"
+import { styled } from "@material-ui/core/styles"
+import { setIn } from "seamless-immutable"
+import Box from "@material-ui/core/Box"
+import Select from "react-select"
+import range from "lodash/range"
+
+const ErrorBox = styled("pre")({
+  color: "red",
+  whiteSpace: "prewrap",
+  fontSize: 11,
+})
+
+const splitOptions = [
+  { label: "2x2", value: "2x2" },
+  { label: "3x3", value: "3x3" },
+  { label: "4x4", value: "4x4" },
+  { label: "3x4", value: "3x4" },
+]
+
+export default ({ open, onChangeDataset, onClose, dataset }) => {
+  const [errors, setErrors] = useState("")
+  const [splitType, setSplitType] = useState(splitOptions[0].value)
+  return (
+    <SimpleDialog
+      open={open}
+      onClose={onClose}
+      title="Convert Image Samples into Segments"
+      actions={[
+        {
+          text: "Convert Image Samples into Segments",
+          onClick: async () => {
+            const [w, h] = splitType.split("x").map((v) => parseInt(v))
+
+            try {
+              onChangeDataset(
+                setIn(
+                  dataset,
+                  ["samples"],
+                  range(h).flatMap((y) =>
+                    range(w).flatMap((x) =>
+                      dataset.samples.map((s) => {
+                        return setIn(s, ["allowedArea"], {
+                          x: x / w,
+                          y: y / h,
+                          width: 1 / w,
+                          height: 1 / h,
+                        })
+                      })
+                    )
+                  )
+                )
+              )
+              onClose()
+            } catch (e) {
+              setErrors(e.toString())
+            }
+          },
+        },
+      ]}
+    >
+      This transformation will multiply the number of image samples you have,
+      splitting each individual image into segments. The image is not processed
+      and clipped, but rather an "allowedArea" property is placed on the sample
+      which limits the labeling to a section of the image. This is useful when
+      trying to reduce the amount of work per image sample.
+      <Box padding={4}>
+        <Select
+          defaultValue={splitOptions[0]}
+          options={splitOptions}
+          onChange={({ value }) => setSplitType(value)}
+        />
+      </Box>
+      {errors && <ErrorBox>{errors}</ErrorBox>}
+    </SimpleDialog>
+  )
+}

src/components/TransformImageSamplesIntoSegmentsDialog/index.js

@@ -16,9 +16,12 @@ import CollectionsIcon from "@material-ui/icons/Collections"
 import TransformVideoFramesToImagesDialog from "../TransformVideoFramesToImagesDialog"
 import usePosthog from "../../utils/use-posthog"
 import TransformLocalFilesToWebURLs from "../TransformLocalFilesToWebURLs"
+import TransformImageSamplesIntoSegmentsDialog from "../TransformImageSamplesIntoSegmentsDialog"
 
 import ComputerIcon from "@material-ui/icons/Computer"
 import LanguageIcon from "@material-ui/icons/Language"
+import CheckBoxOutlineBlankIcon from "@material-ui/icons/CheckBoxOutlineBlank"
+import GridOnIcon from "@material-ui/icons/GridOn"
 
 const ButtonBase = styled(MuiButton)({
   width: 240,
@@ -139,6 +142,13 @@ export default ({ dataset, onChangeDataset }) => {
         >
           Convert Video Frames to Images
         </Button>
+        <Button
+          dialog="split-image-samples-into-segments"
+          Icon1={CheckBoxOutlineBlankIcon}
+          Icon2={GridOnIcon}
+        >
+          Split Image Samples into Segments
+        </Button>
         <TransformVideoKeyframesDialog
           open={selectedDialog === "convert-keyframes-to-samples"}
           onClose={closeDialog}
@@ -169,6 +179,12 @@ export default ({ dataset, onChangeDataset }) => {
           desktopOnly
           onChangeDataset={onChangeDataset}
         ></TransformVideoFramesToImagesDialog>
+        <TransformImageSamplesIntoSegmentsDialog
+          dataset={dataset}
+          open={selectedDialog === "split-image-samples-into-segments"}
+          onClose={closeDialog}
+          onChangeDataset={onChangeDataset}
+        ></TransformImageSamplesIntoSegmentsDialog>
       </div>
     </SelectDialogContext.Provider>
   )