diff --git a/react/src/components/FileUploadManager.tsx b/react/src/components/FileUploadManager.tsx index fe650643f3..faab5551e8 100644 --- a/react/src/components/FileUploadManager.tsx +++ b/react/src/components/FileUploadManager.tsx @@ -12,36 +12,38 @@ import { FileUploadManagerQuery } from 'src/__generated__/FileUploadManagerQuery import { useSuspendedBackendaiClient } from 'src/hooks'; import * as tus from 'tus-js-client'; +type uploadStartFunction = (callbacks?: { + onProgress?: ( + bytesUploaded: number, + bytesTotal: number, + fileName: string, + ) => void; +}) => Promise<{ name: string; bytes: number }>; + type UploadRequest = { vFolderId: string; vFolderName: string; - uploadFilesNames: Array; - startFunctions: Array< - (callbacks?: { - onProgress?: ( - bytesUploaded: number, - bytesTotal: number, - fileName: string, - ) => void; - }) => Promise - >; + uploadFileInfo: Array<{ file: RcFile; startFunction: uploadStartFunction }>; }; -type UploadStatus = { + +type UploadStatusInfo = { vFolderName: string; - pending: Array; - completed: Array; - failed: Array; + pendingFiles: Array; + completedFiles: Array; + failedFiles: Array; + completedBytes: number; + totalExpectedSize: number; }; type UploadStatusMap = { - [vFolderId: string]: UploadStatus; + [vFolderId: string]: UploadStatusInfo; }; -const uploadRequestAtom = atom([]); +const uploadRequestAtom = atom>([]); const uploadStatusAtom = atom({}); const uploadStatusAtomFamily = atomFamily((vFolderId: string) => { return atom( (get) => get(uploadStatusAtom)[vFolderId], - (get, set, newStatus: UploadStatus) => { + (get, set, newStatus: UploadStatusInfo) => { const prev = get(uploadStatusAtom); set(uploadStatusAtom, { ...prev, @@ -50,67 +52,109 @@ const uploadStatusAtomFamily = atomFamily((vFolderId: string) => { }, ); }); + const useUploadStatusAtomStatus = ( vFolderId: string, -): [UploadStatus, (newStatus: UploadStatus) => void] => { +): [UploadStatusInfo, (newStatus: UploadStatusInfo) => void] => { return useAtom(uploadStatusAtomFamily(vFolderId)); }; const FileUploadManager: React.FC = () => { + 'use memo'; + const { t } = useTranslation(); const { upsertNotification } = useSetBAINotification(); const baiClient = useSuspendedBackendaiClient(); const [uploadRequests, setUploadRequests] = useAtom(uploadRequestAtom); const [uploadStatus, setUploadStatus] = useAtom(uploadStatusAtom); - const queue = new PQueue({ concurrency: 1 }); + const queue = new PQueue({ concurrency: 4 }); useEffect(() => { if (uploadRequests.length === 0 || !baiClient) return; uploadRequests.forEach((uploadRequest) => { - const { vFolderId, vFolderName, uploadFilesNames, startFunctions } = - uploadRequest; + const { vFolderId, vFolderName, uploadFileInfo } = uploadRequest; + const currUploadTotalSize = _.sumBy( + uploadFileInfo, + (info) => info.file.size, + ); - setUploadStatus((prev) => ({ - ...prev, - [vFolderId]: { - vFolderName, - pending: [...(prev[vFolderId]?.pending || []), ...uploadFilesNames], - completed: [], - failed: [], - }, - })); + setUploadStatus((prev) => { + const isFirstUpload = !prev[vFolderId]; + const newTotalExpectedSize = + (prev[vFolderId]?.totalExpectedSize || 0) + currUploadTotalSize; + const currPct = isFirstUpload + ? 0 + : ((prev[vFolderId]?.completedBytes || 0) / newTotalExpectedSize) * + 100; - upsertNotification({ - key: 'upload:' + vFolderId, - open: true, - message: t('explorer.UploadToFolder', { - folderName: vFolderName, - }), - backgroundTask: { - status: 'pending', - percent: 0, - onChange: { - pending: t('explorer.ProcessingUpload'), + upsertNotification({ + key: 'upload:' + vFolderId, + open: true, + message: t('explorer.UploadToFolder', { + folderName: vFolderName, + }), + backgroundTask: { + status: 'pending', + percent: currPct, + onChange: { + pending: t('explorer.ProcessingUpload'), + }, }, - }, - duration: 0, + duration: 0, + }); + + return { + ...prev, + [vFolderId]: { + vFolderName, + pendingFiles: [ + ...(prev[vFolderId]?.pendingFiles || []), + ...uploadFileInfo.map( + (info) => info.file.webkitRelativePath || info.file.name, + ), + ], + completedFiles: prev[vFolderId]?.completedFiles || [], + failedFiles: prev[vFolderId]?.failedFiles || [], + completedBytes: prev[vFolderId]?.completedBytes || 0, + totalExpectedSize: newTotalExpectedSize, + }, + }; }); - startFunctions.forEach((startFunction) => { + uploadFileInfo.forEach(({ startFunction }) => { queue.add(async () => { + let previousBytesUploaded = 0; await startFunction({ - onProgress: (bytesUploaded, bytesTotal, fileName) => { + onProgress: (bytesUploaded, _bytesTotal, fileName) => { + // Since bytesUploaded is cumulative, calculate delta from previous value + const deltaBytes = bytesUploaded - previousBytesUploaded; + previousBytesUploaded = bytesUploaded; + setUploadStatus((prev) => { - const remainingFiles = prev[vFolderId]?.pending || []; + const uploadedFilesCount = + prev[vFolderId]?.completedFiles?.length || 0; + const totalUploadedFilesCount = + (prev[vFolderId]?.completedFiles?.length || 0) + + (prev[vFolderId]?.failedFiles?.length || 0) + + (prev[vFolderId]?.pendingFiles?.length || 0); + + const totalExpectedSize = + prev[vFolderId]?.totalExpectedSize || 0; + const currentCompletedBytes = + (prev[vFolderId]?.completedBytes || 0) + deltaBytes; + upsertNotification({ key: 'upload:' + vFolderId, message: `${t('explorer.UploadToFolder', { folderName: vFolderName, - })}${remainingFiles.length > 1 ? ` (${remainingFiles.length})` : ''}`, + })}${` (${uploadedFilesCount} / ${totalUploadedFilesCount})`}`, backgroundTask: { status: 'pending', - percent: Math.round((bytesUploaded / bytesTotal) * 100) - 1, + percent: + totalExpectedSize > 0 + ? (currentCompletedBytes / totalExpectedSize) * 100 + : 0, onChange: { pending: t('explorer.FileInProgress', { fileName: fileName, @@ -119,31 +163,45 @@ const FileUploadManager: React.FC = () => { }, }); - return prev; + // handle upload progress bytes update + return { + ...prev, + [vFolderId]: { + ...prev[vFolderId], + completedBytes: currentCompletedBytes, + }, + }; }); }, }) - .then((fileName: string) => { + // handle uploaded file name only, size is already handled in progress + .then(({ name: fileName }) => { setUploadStatus((prev) => ({ ...prev, [vFolderId]: { ...prev[vFolderId], - pending: prev[vFolderId].pending.filter( - (f) => f !== fileName, + pendingFiles: prev[vFolderId].pendingFiles.filter( + (f: string) => f !== fileName, ), - completed: [...prev[vFolderId].completed, fileName], + completedFiles: [ + ...(prev[vFolderId]?.completedFiles || []), + fileName, + ], }, })); }) - .catch((fileName: string) => { + .catch(({ name: fileName }) => { setUploadStatus((prev) => ({ ...prev, [vFolderId]: { ...prev[vFolderId], - pending: prev[vFolderId].pending.filter( - (f) => f !== fileName, + pendingFiles: prev[vFolderId].pendingFiles.filter( + (f: string) => f !== fileName, ), - failed: [...prev[vFolderId].failed, fileName], + failedFiles: [ + ...(prev[vFolderId]?.failedFiles || []), + fileName, + ], }, })); }); @@ -156,9 +214,9 @@ const FileUploadManager: React.FC = () => { useEffect(() => { Object.entries(uploadStatus).forEach(([vFolderId, status]) => { - if (!_.isEmpty(status?.pending)) return; + if (!_.isEmpty(status?.pendingFiles)) return; - if (!_.isEmpty(status?.failed)) { + if (!_.isEmpty(status?.failedFiles)) { upsertNotification({ key: 'upload:' + vFolderId, open: true, @@ -174,9 +232,9 @@ const FileUploadManager: React.FC = () => { }), }, }, - extraDescription: _.join(status?.failed, ', '), + extraDescription: _.join(status?.failedFiles, ', '), }); - } else if (!_.isEmpty(status?.completed)) { + } else if (!_.isEmpty(status?.completedFiles)) { upsertNotification({ key: 'upload:' + vFolderId, open: true, @@ -196,7 +254,9 @@ const FileUploadManager: React.FC = () => { ...prev, [vFolderId]: { ...prev[vFolderId], - completed: [], + completedFiles: [], + completedBytes: 0, + totalExpectedSize: 0, }, })); } @@ -210,6 +270,8 @@ const FileUploadManager: React.FC = () => { export default FileUploadManager; export const useFileUploadManager = (vFolderId: string) => { + 'use memo'; + const baiClient = useConnectedBAIClient(); const { t } = useTranslation(); const { upsertNotification } = useSetBAINotification(); @@ -270,9 +332,9 @@ export const useFileUploadManager = (vFolderId: string) => { ) => { if (!validateUploadRequest(files, vfolderId)) return; - const uploadFileNames: Array = []; + const fileToUpload: Array = []; const startUploadFunctionMap = _.map(files, (file) => { - uploadFileNames.push(file.webkitRelativePath || file.name); + fileToUpload.push(file); return async (callbacks?: { onProgress?: ( bytesUploaded: number, @@ -289,37 +351,51 @@ export const useFileUploadManager = (vFolderId: string) => { vfolderId, ); - return new Promise((resolve, reject) => { - const upload = new tus.Upload(file, { - endpoint: uploadUrl, - uploadUrl: uploadUrl, - retryDelays: [0, 3000, 5000, 10000, 20000], - chunkSize: getOptimalChunkSize(file.size), - storeFingerprintForResuming: false, // Disable localStorage storage - metadata: { - filename: file.name, - filetype: file.type, - }, - onProgress: (bytesUploaded, bytesTotal) => { - callbacks?.onProgress?.(bytesUploaded, bytesTotal, file.name); - }, - onSuccess: () => { - resolve(file.webkitRelativePath || file.name); - }, - onError: () => { - reject(file.webkitRelativePath || file.name); - }, - }); - upload.start(); - }); + return new Promise<{ name: string; bytes: number }>( + (resolve, reject) => { + const upload = new tus.Upload(file, { + endpoint: uploadUrl, + uploadUrl: uploadUrl, + retryDelays: [0, 3000, 5000, 10000, 20000], + chunkSize: getOptimalChunkSize(file.size), + storeFingerprintForResuming: false, // Disable localStorage storage + metadata: { + filename: file.name, + filetype: file.type, + }, + onProgress: (bytesUploaded, bytesTotal) => { + callbacks?.onProgress?.(bytesUploaded, bytesTotal, file.name); + }, + onSuccess: () => { + resolve({ + name: file.webkitRelativePath || file.name, + bytes: file.size, + }); + }, + onError: () => { + reject({ + name: file.webkitRelativePath || file.name, + bytes: file.size, + }); + }, + }); + upload.start(); + }, + ); }; }); const uploadRequestInfo: UploadRequest = { vFolderId: vfolderId, vFolderName: vfolder_node?.name ?? '', - uploadFilesNames: uploadFileNames, - startFunctions: startUploadFunctionMap, + uploadFileInfo: _.zipWith( + fileToUpload, + startUploadFunctionMap, + (file, startFunction) => ({ + file, + startFunction, + }), + ), }; setUploadRequests((prev) => [...prev, uploadRequestInfo]); }; diff --git a/react/src/components/FolderExplorerModal.tsx b/react/src/components/FolderExplorerModal.tsx index 5915f5fe12..5d29970746 100644 --- a/react/src/components/FolderExplorerModal.tsx +++ b/react/src/components/FolderExplorerModal.tsx @@ -69,7 +69,7 @@ const FolderExplorerModal: React.FC = ({ const bodyRef = useRef(null); useEffect(() => { - if (uploadStatus && _.isEmpty(uploadStatus.pending)) { + if (uploadStatus && _.isEmpty(uploadStatus?.pendingFiles)) { updateFetchKey(); } }, [uploadStatus, updateFetchKey]);