787 enable uploading multiple files at once (#802)

* endpoint does not work, but new endpoint for multiple files added.
tests also added.
note - i will consolidate these endpoints once it is working

* iterate through files, upload all of them

* tests sort of work, need to be cleaned up
New Menu has multiple file uploads, does not work yet

* multiple property now in, new views, does not upload yet

* multiple select selects shows multiple files when they are selected together
it does not when they are selected one after the other

* logging files when we click finish

* codegen, adding multiple files endpoint

* getting a 422 for uploading multiple files

* does not work

* adding print statement back in

* adding print statement back in

* some changes to how formdata is handled

error:

"WARNING:multipart.multipart:Did not find CR at end of boundary (40)"

* clean up

* 734 view and modify list of metadata definitions in UI (#758)

* add endpoint and test

* codegen and black

* add action

* add search and get endpoint

* add search and get endpoint to action

* add to reducer

* basic page for metadata definition ready

* add delete confirmation

* add create metadata modal

* ordering in decending order

* update message

* black

* fix the pytest

* codegen

* fix codegen

* stretched icon when extractor description long (#789)

* disable ripple effect

* fix the stretch

---------

Co-authored-by: Chen Wang <[email protected]>

* 792 clear previous log before switching extraction logs (#793)

* immediately update before the time interval refresh

* add proper reset

* Pagination for files & folder page under dataset (#797)

* Implement wordcloud visualization (#786)

* Implement wordcloud visualization

* Changing the name

* message if no datasets exist, button link to create (#767)

* button to create datasets if none exist

* new  message

* center, no previous and next if we have no datasets

* Updated the labels for Share (#798)

* Updated the labels for Share

* Fixed the message

* 778 page to display each metadata definition (#801)

* new page of metadata definition entry

* initial page for metadata definition page

* basic metadata defintion page

* 788 duplicated extractor registration when extractor version updated (#791)

* we now replace the old extractor

* created timestamp is from original extractor

* 701 improve file version selection (#743)

* placeholder for where new version select will go

* not clickable, new imports

* select no longer in version chip
new dropdown on filepage
problem - current version not selected on file load

* does not look good
select and file details end up at bottom

* clean up the logic

* fix bug

* make it look nicer

* add snackbar

* change to half width

* change history to details

* remove width

---------

Co-authored-by: Chen Wang <[email protected]>

* add swagger to traefik (#805)

* add swagger to traefik

* add to deployment

* add to ingress rule

* Change hostname to edu (#813)

* updated

* typo

* missed more spots

* basic message for no metadata definitions (#766)

* endpoint logic is wrong; also fix pytest

* write better pytest

* rewrite the structure of select file and construct the form in the function

* back to bad request again

* everything working except redirect

* go to the first files route

* formatting

* codegen

* fixing delete of files in tests

* fixing package lock

---------

Co-authored-by: Chen Wang <[email protected]>
Co-authored-by: Dipannita <[email protected]>

Author: 3 people

backend/app/routers/datasets.py

@@ -9,22 +9,21 @@
 from typing import List, Optional
 
 from beanie import PydanticObjectId
-from beanie.operators import Or
 from beanie.odm.operators.update.general import Inc
+from beanie.operators import Or
 from bson import ObjectId
 from bson import json_util
 from elasticsearch import Elasticsearch
 from fastapi import (
     APIRouter,
     HTTPException,
     Depends,
-    Security,
     File,
     UploadFile,
     Request,
 )
 from fastapi.responses import StreamingResponse
-from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from fastapi.security import HTTPBearer
 from minio import Minio
 from pika.adapters.blocking_connection import BlockingChannel
 from rocrate.model.person import Person
@@ -51,8 +50,8 @@
 from app.models.folders import FolderOut, FolderIn, FolderDB, FolderDBViewList
 from app.models.metadata import MetadataDB
 from app.models.pyobjectid import PyObjectId
-from app.models.users import UserOut
 from app.models.thumbnails import ThumbnailDB
+from app.models.users import UserOut
 from app.rabbitmq.listeners import submit_dataset_job
 from app.routers.files import add_file_entry, remove_file_entry
 from app.search.connect import (
@@ -462,6 +461,54 @@ async def save_file(
     raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found")
 
 
+@router.post("/{dataset_id}/filesMultiple", response_model=List[FileOut])
+async def save_files(
+    dataset_id: str,
+    files: List[UploadFile],
+    folder_id: Optional[str] = None,
+    user=Depends(get_current_user),
+    fs: Minio = Depends(dependencies.get_fs),
+    es=Depends(dependencies.get_elasticsearchclient),
+    rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq),
+    allow: bool = Depends(Authorization("uploader")),
+):
+    if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
+        files_added = []
+        for file in files:
+            if user is None:
+                raise HTTPException(
+                    status_code=401,
+                    detail=f"User not found. Session might have expired.",
+                )
+
+            new_file = FileDB(name=file.filename, creator=user, dataset_id=dataset.id)
+
+            if folder_id is not None:
+                if (
+                    folder := await FolderDB.get(PydanticObjectId(folder_id))
+                ) is not None:
+                    new_file.folder_id = folder.id
+                else:
+                    raise HTTPException(
+                        status_code=404, detail=f"Folder {folder_id} not found"
+                    )
+
+            await add_file_entry(
+                new_file,
+                user,
+                fs,
+                es,
+                rabbitmq_client,
+                file.file,
+                content_type=file.content_type,
+            )
+            files_added.append(new_file.dict())
+        return files_added
+
+    else:
+        raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found")
+
+
 @router.post("/createFromZip", response_model=DatasetOut)
 async def create_dataset_from_zip(
     user=Depends(get_current_user),

backend/app/tests/test_files.py

@@ -3,18 +3,26 @@
 from fastapi.testclient import TestClient
 
 from app.config import settings
-from app.tests.utils import create_dataset, upload_file, generate_png
+from app.tests.utils import create_dataset, upload_file, upload_files, generate_png
 
 
 def test_create_and_delete(client: TestClient, headers: dict):
     dataset_id = create_dataset(client, headers).get("id")
     response = upload_file(client, headers, dataset_id)
-    file = response
     file_id = response["id"]
     # DELETE FILE
     response = client.delete(f"{settings.API_V2_STR}/files/{file_id}", headers=headers)
     assert response.status_code == 200
 
+    response_multiple = upload_files(client, headers, dataset_id)
+    file_ids = [f["id"] for f in response_multiple]
+    # DELETE MULTIPLE FILES
+    for file_id in file_ids:
+        response = client.delete(
+            f"{settings.API_V2_STR}/files/{file_id}", headers=headers
+        )
+        assert response.status_code == 200
+
 
 def test_get_one(client: TestClient, headers: dict):
     temp_name = "testing file.txt"

backend/app/tests/utils.py

@@ -33,8 +33,11 @@
     "description": "a dataset is a container of files and metadata",
 }
 
-filename_example = "test_upload.csv"
-file_content_example = "year,location,count\n2023,Atlanta,4"
+filename_example_1 = "test_upload1.csv"
+file_content_example_1 = "year,location,count\n2023,Atlanta,4"
+
+filename_example_2 = "test_upload2.csv"
+file_content_example_2 = "year,location,count\n2022,Seattle,2"
 
 listener_v2_example = {
     "name": "test.listener_v2_example",
@@ -143,8 +146,8 @@ def upload_file(
     client: TestClient,
     headers: dict,
     dataset_id: str,
-    filename=filename_example,
-    content=file_content_example,
+    filename=filename_example_1,
+    content=file_content_example_1,
 ):
     """Uploads a dummy file (optionally with custom name/content) to a dataset and returns the JSON."""
     with open(filename, "w") as tempf:
@@ -161,6 +164,36 @@ def upload_file(
     return response.json()
 
 
+def upload_files(
+    client: TestClient,
+    headers: dict,
+    dataset_id: str,
+    filenames=[filename_example_1, filename_example_2],
+    file_contents=[file_content_example_1, file_content_example_2],
+):
+    """Uploads a dummy file (optionally with custom name/content) to a dataset and returns the JSON."""
+    upload_files = []
+    for i in range(0, len(filenames)):
+        with open(filenames[i], "w") as tempf:
+            tempf.write(file_contents[i])
+            upload_files.append(filenames[i])
+    files = [
+        ("files", open(filename_example_1, "rb")),
+        ("files", open(filename_example_2, "rb")),
+    ]
+    response = client.post(
+        f"{settings.API_V2_STR}/datasets/{dataset_id}/filesMultiple",
+        headers=headers,
+        files=files,
+    )
+    for f in upload_files:
+        os.remove(f)
+    assert response.status_code == 200
+    json_response = response.json()
+    assert len(json_response) == 2
+    return response.json()
+
+
 def create_folder(
     client: TestClient,
     headers: dict,

frontend/src/actions/file.js

@@ -146,6 +146,42 @@ export function createFile(selectedDatasetId, folderId, selectedFile) {
 	};
 }
 
+export const CREATE_FILES = "CREATE_FILES";
+
+export function createFiles(selectedDatasetId, selectedFiles, folderId) {
+	return (dispatch) => {
+		let formData = new FormData();
+		let tmp = [];
+		if (selectedFiles.length > 0) {
+			for (let i = 0; i < selectedFiles.length; i++) {
+				tmp.push(selectedFiles[i]);
+			}
+		}
+		formData["files"] = tmp;
+
+		return V2.DatasetsService.saveFilesApiV2DatasetsDatasetIdFilesMultiplePost(
+			selectedDatasetId,
+			formData,
+			folderId
+		)
+			.then((files) => {
+				dispatch({
+					type: CREATE_FILES,
+					files: files,
+					receivedAt: Date.now(),
+				});
+			})
+			.catch((reason) => {
+				dispatch(
+					handleErrors(
+						reason,
+						createFiles(selectedDatasetId, selectedFiles, folderId)
+					)
+				);
+			});
+	};
+}
+
 export const RESET_CREATE_FILE = "RESET_CREATE_FILE";
 
 export function resetFileCreated() {
@@ -157,6 +193,17 @@ export function resetFileCreated() {
 	};
 }
 
+export const RESET_CREATE_FILES = "RESET_CREATE_FILES";
+
+export function resetFilesCreated() {
+	return (dispatch) => {
+		dispatch({
+			type: RESET_CREATE_FILES,
+			receivedAt: Date.now(),
+		});
+	};
+}
+
 export const UPDATE_FILE = "UPDATE_FILE";
 
 export function updateFile(selectedFile, fileId) {

frontend/src/components/datasets/NewMenu.tsx

@@ -13,6 +13,7 @@ import React from "react";
 import { useSelector } from "react-redux";
 import { RootState } from "../../types/data";
 import { UploadFile } from "../files/UploadFile";
+import { UploadFileMultiple } from "../files/UploadFileMultiple";
 import UploadIcon from "@mui/icons-material/Upload";
 import { Folder } from "@material-ui/icons";
 
@@ -29,6 +30,8 @@ export const NewMenu = (props: ActionsMenuProps): JSX.Element => {
 
 	const [anchorEl, setAnchorEl] = React.useState<Element | null>(null);
 	const [createFileOpen, setCreateFileOpen] = React.useState<boolean>(false);
+	const [createMultipleFileOpen, setCreateMultipleFileOpen] =
+		React.useState<boolean>(false);
 	const [newFolder, setNewFolder] = React.useState<boolean>(false);
 
 	const handleCloseNewFolder = () => {
@@ -58,6 +61,22 @@ export const NewMenu = (props: ActionsMenuProps): JSX.Element => {
 					folderId={folderId}
 				/>
 			</Dialog>
+			<Dialog
+				open={createMultipleFileOpen}
+				onClose={() => {
+					setCreateMultipleFileOpen(false);
+				}}
+				fullWidth={true}
+				maxWidth="lg"
+				aria-labelledby="form-dialog"
+			>
+				<UploadFileMultiple
+					selectedDatasetId={datasetId}
+					selectedDatasetName={about.name}
+					setCreateMultipleFileOpen={setCreateMultipleFileOpen}
+					folderId={folderId}
+				/>
+			</Dialog>
 
 			<CreateFolder
 				datasetId={datasetId}
@@ -92,6 +111,17 @@ export const NewMenu = (props: ActionsMenuProps): JSX.Element => {
 					</ListItemIcon>
 					<ListItemText>Upload File</ListItemText>
 				</MenuItem>
+				<MenuItem
+					onClick={() => {
+						setCreateMultipleFileOpen(true);
+						handleOptionClose();
+					}}
+				>
+					<ListItemIcon>
+						<UploadIcon fontSize="small" />
+					</ListItemIcon>
+					<ListItemText>Upload Multiple Files</ListItemText>
+				</MenuItem>
 				<MenuItem
 					onClick={() => {
 						setNewFolder(true);

frontend/src/components/files/UploadFileInputMultiple.tsx

@@ -0,0 +1,35 @@
+import React from "react";
+
+import { Box, Input } from "@mui/material";
+
+type UploadFileMultipleModalProps = {
+	setSelectedFiles: any;
+};
+
+// https://stackoverflow.com/questions/68213700/react-js-upload-multiple-files
+export const UploadFileInputMultiple: React.FC<UploadFileMultipleModalProps> = (
+	props: UploadFileMultipleModalProps
+) => {
+	const { setSelectedFiles } = props;
+
+	const handleMultipleFileChange = (event) => {
+		// let tempFormData = new FormData();
+		// for (let i = 0; i < event.target.files.length; i++) {
+		// 	tempFormData.append("files", event.target.files[i]);
+		// }
+		setSelectedFiles(event.target.files);
+	};
+
+	return (
+		<Box sx={{ width: "100%", padding: "1em 0em" }}>
+			<Input
+				id="file-input-multiple"
+				type="file"
+				inputProps={{ multiple: true }}
+				onChange={handleMultipleFileChange}
+				sx={{ width: "100%" }}
+				disableUnderline={true}
+			/>
+		</Box>
+	);
+};