fix: Merge pull request #255 from UniversalDataTool/feat/s3-import

IAM-based S3 Data Import and Upload

Author: seveibar

package.json

@@ -83,13 +83,15 @@
     "any-shell-escape": "^0.1.1",
     "aws-amplify": "^3.0.6",
     "aws-sign": "^1.0.0",
+    "aws4fetch": "^1.0.12",
     "axios": "^0.19.2",
     "bent": "^7.1.0",
     "brace": "^0.11.1",
     "detect-browser": "^5.0.0",
     "download-file": "^0.1.5",
     "duration": "^0.2.2",
     "fast-json-patch": "^3.0.0-1",
+    "fast-xml-parser": "^3.17.4",
     "ffmpeg-static": "^4.2.4",
     "form-data": "^3.0.0",
     "i18next": "^19.4.4",
@@ -102,7 +104,7 @@
     "posthog-js": "^1.0.4",
     "react-ace": "^7.0.4",
     "react-data-table-component": "^6.2.2",
-    "react-dropzone": "^10.1.8",
+    "react-dropzone": "^11.0.3",
     "react-github-btn": "^1.2.0",
     "react-hotkeys": "^2.0.0",
     "react-i18next": "^11.4.0",

src/components/AddAuthFromTemplateDialog/index.js

@@ -102,6 +102,13 @@ const forms = {
         type: "text",
         isRequired: true,
       },
+      {
+        name: "auth.s3iam.region",
+        title: "Region",
+        placeholder: "us-east-1",
+        type: "text",
+        isRequired: true,
+      },
     ],
   },
   proxy: {

src/components/AppConfig/index.js

@@ -14,6 +14,7 @@ const configKeyNames = [
   "auth.cognito.storage.aws_s3.region",
   "auth.s3iam.access_key_id",
   "auth.s3iam.secret_access_key",
+  "auth.s3iam.region",
   "auth.proxy.corsproxy",
   "labelhelp.disabled",
   "labelhelp.apikey",
@@ -27,9 +28,20 @@ const defaultAppConfig = {
     "https://corsproxy.seve.workers.dev/corsproxy/?apiurl={URL}",
 }
 
+const jsonParseOrEmpty = (s) => {
+  try {
+    return JSON.parse(s)
+  } catch (e) {
+    return {}
+  }
+}
+
 // NOTE: appConfig should not allow any nested values
 export const AppConfigContext = createContext({
-  appConfig: {},
+  appConfig: {
+    ...defaultAppConfig,
+    ...jsonParseOrEmpty(window.localStorage.app_config),
+  },
   setAppConfig: (newConfig) => undefined,
   fromConfig: (key) => undefined,
   setInConfig: (key, value) => undefined,

src/components/ImportFromS3Dialog/index.js

@@ -1,13 +1,90 @@
-import React from "react"
+import React, { useState, useEffect } from "react"
+import Box from "@material-ui/core/Box"
+import CircularProgress from "@material-ui/core/CircularProgress"
 import SimpleDialog from "../SimpleDialog"
+import S3PathSelector from "../S3PathSelector"
+import getSampleFromUrl from "../PasteUrlsDialog/get-sample-from-url.js"
+// import { useAppConfig } from "../AppConfig"
+import useIAMS3API from "../../utils/auth-handlers/use-iam-s3-api"
 
-export const ImportFromS3Dialog = ({ open, onClose }) => {
+export const ImportFromS3Dialog = ({ open, onClose, onAddSamples }) => {
+  const { listBuckets, listBucketItemsAt } = useIAMS3API()
+  const [s3Path, setS3Path] = useState("")
+  const [{ options, optionsLoading }, setOptions] = useState({
+    optionsLoading: true,
+  })
+  useEffect(() => {
+    if (!open) return
+    if (!listBuckets) return
+    async function loadS3Path() {
+      if (s3Path === "") {
+        setOptions({
+          optionsLoading: false,
+          options: (
+            await listBuckets()
+          ).ListAllMyBucketsResult.Buckets.Bucket.map((b) => ({
+            name: b.Name,
+            type: "bucket",
+          })),
+        })
+      } else {
+        setOptions({
+          optionsLoading: false,
+          options: (
+            (await listBucketItemsAt(s3Path)).ListBucketResult.CommonPrefixes ||
+            []
+          ).map((a) => ({
+            name: a.Prefix.split("/").slice(-2).join("/"),
+            type: "directory",
+          })),
+        })
+      }
+    }
+    loadS3Path()
+    // eslint-disable-next-line
+  }, [s3Path, listBuckets, listBucketItemsAt, open])
+  if (!open) return null
   return (
-    <SimpleDialog onClose={onClose} open={open} title="Import from S3">
-      This is a work in progress! Available soon!{" "}
-      <a href="https://github.com/UniversalDataTool/universal-data-tool/issues/205">
-        Track the progress here.
-      </a>
+    <SimpleDialog
+      onClose={onClose}
+      open={open}
+      title="Import from S3"
+      actions={[
+        {
+          onClick: async () => {
+            const bucket = s3Path.match(/s3:\/\/([^/]+)/)[1]
+
+            // TODO support more than 1000 file import by using
+            // continuation-token on S3
+            const items = (
+              await listBucketItemsAt(s3Path, { files: true })
+            ).ListBucketResult.Contents.filter(
+              (item) => !item.Key.endsWith("/")
+            )
+              .map((item) => `https://s3.amazonaws.com/${bucket}/${item.Key}`)
+              .map((url) => getSampleFromUrl(url, { returnNulls: true }))
+              .filter(Boolean)
+            onAddSamples(items)
+          },
+          disabled: !s3Path,
+          text: "Import All Files in Directory",
+        },
+      ]}
+    >
+      {optionsLoading ? (
+        <Box height={300} width={400} paddingTop={4} textAlign="center">
+          <CircularProgress size={100} />
+        </Box>
+      ) : (
+        <S3PathSelector
+          currentPath={s3Path}
+          options={options || []}
+          onChangePath={(newS3Path) => {
+            setS3Path(newS3Path)
+            setOptions({ optionsLoading: true })
+          }}
+        />
+      )}
     </SimpleDialog>
   )
 }

src/components/ImportFromS3Dialog/index.story.js

@@ -0,0 +1,16 @@
+// @flow
+
+import React from "react"
+
+import { storiesOf } from "@storybook/react"
+import { action } from "@storybook/addon-actions"
+
+import ImportFromS3Dialog from "./"
+
+// NOTE: THIS STORY REQUIRES the localStorage app_config to have s3 auth options!! It won't
+// work by default!
+
+storiesOf("ImportFromS3Dialog", module).add(
+  "Basic (doesn't work by default)",
+  () => <ImportFromS3Dialog open onAddSamples={action("onAddSamples")} />
+)

src/components/MultiFileDrop/index.js

@@ -0,0 +1,86 @@
+import React, { useCallback, useState } from "react"
+import { styled } from "@material-ui/core/styles"
+import * as colors from "@material-ui/core/colors"
+import Box from "@material-ui/core/Box"
+import { useDropzone } from "react-dropzone"
+
+const Container = styled(Box)({
+  width: 400,
+  height: 200,
+  padding: 20,
+  border: `2px dotted ${colors.grey[400]}`,
+  textAlign: "center",
+  fontSize: 24,
+})
+
+const FileCounter = styled(Box)({
+  fontSize: 24,
+  color: colors.grey[700],
+  fontWeight: "bold",
+})
+
+/*
+  This component allows you to upload multiple files, then reads the files in
+  serial. It's perfect for an uploading process where you don't want to upload one
+  file at a time. It gives a progress bar indicating upload progres.
+*/
+
+export const MultiFileDrop = ({ loadFile, onComplete }) => {
+  const [totalUploaded, setTotalUploaded] = useState(0)
+  const [errors, setErrors] = useState([])
+  const [{ totalFiles, filesSelected }, setUploadStartState] = useState({
+    filesSelected: false,
+  })
+  const onDrop = useCallback(async (acceptedFiles) => {
+    setUploadStartState({
+      totalFiles: acceptedFiles.length,
+      filesSelected: true,
+    })
+    const errors = []
+    for (const [index, file] of acceptedFiles.entries()) {
+      await loadFile(file).catch((err) => {
+        errors.push(err.toString())
+        setErrors(errors)
+      })
+      setTotalUploaded(index + 1)
+    }
+    onComplete()
+    // eslint-disable-next-line
+  }, [])
+  const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop })
+
+  if (!filesSelected) {
+    return (
+      <Container {...getRootProps()}>
+        <input {...getInputProps()} />
+        {isDragActive ? (
+          <p>Drop the files here ...</p>
+        ) : (
+          <p>Drag 'n' drop some files here, or click to select files</p>
+        )}
+      </Container>
+    )
+  }
+
+  return (
+    <Container>
+      <FileCounter>
+        {totalUploaded} / {totalFiles} Files Processed
+      </FileCounter>
+      {/* TODO progress bar */}
+      <Box
+        maxHeight={100}
+        overflowY="auto"
+        textAlign="left"
+        color="red"
+        fontSize={12}
+      >
+        {errors.map((err, i) => (
+          <div key={i}>{err}</div>
+        ))}
+      </Box>
+    </Container>
+  )
+}
+
+export default MultiFileDrop

src/components/PasteUrlsDialog/get-sample-from-url.js

@@ -0,0 +1,29 @@
+export default (s, opts = {}) => {
+  console.log(s)
+  let extension = s.replace(/\?.*/g, "").split(".").slice(-1)[0]
+  if (s.includes("gstatic.com/images")) {
+    extension = "jpg"
+  }
+  switch (extension.toLowerCase()) {
+    case "png":
+    case "jpg":
+    case "gif":
+    case "jpeg":
+    case "bmp": {
+      return { imageUrl: s }
+    }
+    case "pdf": {
+      return { pdfUrl: s }
+    }
+    case "mp3":
+    case "wav": {
+      return { audioUrl: s }
+    }
+    default: {
+      if (opts.returnNulls) return null
+      throw new Error(`extension not recognized: "${extension}" in "${s}"`)
+      // TODO if the user doesn't care, return null (this
+      // behavior could be enabled with textfield option)
+    }
+  }
+}

src/components/PasteUrlsDialog/index.js

@@ -4,6 +4,7 @@ import { styled } from "@material-ui/core/styles"
 import SimpleDialog from "../SimpleDialog"
 import Select from "react-select"
 import * as colors from "@material-ui/core/colors"
+import getSampleFromUrl from "./get-sample-from-url"
 
 const SelectContainer = styled("div")({
   marginTop: 8,
@@ -48,41 +49,11 @@ export default ({ open, onClose, onAddSamples }) => {
                   .split("\n")
                   .map((l) => l.trim())
                   .filter(Boolean)
-                  .map((s) => {
-                    if (urlType === "images") {
-                      return { imageUrl: s }
-                    }
-                    let extension = s
-                      .replace(/\?.*/g, "")
-                      .split(".")
-                      .slice(-1)[0]
-                    if (s.includes("gstatic.com/images")) {
-                      extension = "jpg"
-                    }
-                    switch (extension.toLowerCase()) {
-                      case "png":
-                      case "jpg":
-                      case "gif":
-                      case "jpeg":
-                      case "bmp": {
-                        return { imageUrl: s }
-                      }
-                      case "pdf": {
-                        return { pdfUrl: s }
-                      }
-                      case "mp3":
-                      case "wav": {
-                        return { audioUrl: s }
-                      }
-                      default: {
-                        throw new Error(
-                          `extension not recognized: "${extension}" in "${s}"`
-                        )
-                        // TODO if the user doesn't care, return null (this
-                        // behavior could be enabled with textfield option)
-                      }
-                    }
-                  })
+                  .map((s) =>
+                    urlType === "images"
+                      ? { imageUrl: s }
+                      : getSampleFromUrl(s, { returnNulls: true })
+                  )
                   .filter(Boolean)
               )
             } catch (e) {