Skip to content

fix(FR-1501): fix file upload status tracking and error handling #4412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+166 −90
Open

fix(FR-1501): fix file upload status tracking and error handling #4412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
254 changes: 165 additions & 89 deletions react/src/components/FileUploadManager.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,36 +12,38 @@ import { FileUploadManagerQuery } from 'src/__generated__/FileUploadManagerQuery
import { useSuspendedBackendaiClient } from 'src/hooks';
import * as tus from 'tus-js-client';

type uploadStartFunction = (callbacks?: {
onProgress?: (
bytesUploaded: number,
bytesTotal: number,
fileName: string,
) => void;
}) => Promise<{ name: string; bytes: number }>;

type UploadRequest = {
vFolderId: string;
vFolderName: string;
uploadFilesNames: Array<string>;
startFunctions: Array<
(callbacks?: {
onProgress?: (
bytesUploaded: number,
bytesTotal: number,
fileName: string,
) => void;
}) => Promise<string>
>;
uploadFileInfo: Array<{ file: RcFile; startFunction: uploadStartFunction }>;
};
type UploadStatus = {

type UploadStatusInfo = {
vFolderName: string;
pending: Array<string>;
completed: Array<string>;
failed: Array<string>;
pendingFiles: Array<string>;
completedFiles: Array<string>;
failedFiles: Array<string>;
completedBytes: number;
totalExpectedSize: number;
};
type UploadStatusMap = {
[vFolderId: string]: UploadStatus;
[vFolderId: string]: UploadStatusInfo;
};

const uploadRequestAtom = atom<UploadRequest[]>([]);
const uploadRequestAtom = atom<Array<UploadRequest>>([]);
const uploadStatusAtom = atom<UploadStatusMap>({});
const uploadStatusAtomFamily = atomFamily((vFolderId: string) => {
return atom(
(get) => get(uploadStatusAtom)[vFolderId],
(get, set, newStatus: UploadStatus) => {
(get, set, newStatus: UploadStatusInfo) => {
const prev = get(uploadStatusAtom);
set(uploadStatusAtom, {
...prev,
Expand All @@ -50,67 +52,109 @@ const uploadStatusAtomFamily = atomFamily((vFolderId: string) => {
},
);
});

const useUploadStatusAtomStatus = (
vFolderId: string,
): [UploadStatus, (newStatus: UploadStatus) => void] => {
): [UploadStatusInfo, (newStatus: UploadStatusInfo) => void] => {
return useAtom(uploadStatusAtomFamily(vFolderId));
};

const FileUploadManager: React.FC = () => {
'use memo';

const { t } = useTranslation();
const { upsertNotification } = useSetBAINotification();
const baiClient = useSuspendedBackendaiClient();
const [uploadRequests, setUploadRequests] = useAtom(uploadRequestAtom);
const [uploadStatus, setUploadStatus] = useAtom(uploadStatusAtom);
const queue = new PQueue({ concurrency: 1 });
const queue = new PQueue({ concurrency: 4 });

useEffect(() => {
if (uploadRequests.length === 0 || !baiClient) return;

uploadRequests.forEach((uploadRequest) => {
const { vFolderId, vFolderName, uploadFilesNames, startFunctions } =
uploadRequest;
const { vFolderId, vFolderName, uploadFileInfo } = uploadRequest;
const currUploadTotalSize = _.sumBy(
uploadFileInfo,
(info) => info.file.size,
);

setUploadStatus((prev) => ({
...prev,
[vFolderId]: {
vFolderName,
pending: [...(prev[vFolderId]?.pending || []), ...uploadFilesNames],
completed: [],
failed: [],
},
}));
setUploadStatus((prev) => {
const isFirstUpload = !prev[vFolderId];
const newTotalExpectedSize =
(prev[vFolderId]?.totalExpectedSize || 0) + currUploadTotalSize;
const currPct = isFirstUpload
? 0
: ((prev[vFolderId]?.completedBytes || 0) / newTotalExpectedSize) *
100;

upsertNotification({
key: 'upload:' + vFolderId,
open: true,
message: t('explorer.UploadToFolder', {
folderName: vFolderName,
}),
backgroundTask: {
status: 'pending',
percent: 0,
onChange: {
pending: t('explorer.ProcessingUpload'),
upsertNotification({
key: 'upload:' + vFolderId,
open: true,
message: t('explorer.UploadToFolder', {
folderName: vFolderName,
}),
backgroundTask: {
status: 'pending',
percent: currPct,
onChange: {
pending: t('explorer.ProcessingUpload'),
},
},
},
duration: 0,
duration: 0,
});

return {
...prev,
[vFolderId]: {
vFolderName,
pendingFiles: [
...(prev[vFolderId]?.pendingFiles || []),
...uploadFileInfo.map(
(info) => info.file.webkitRelativePath || info.file.name,
),
],
completedFiles: prev[vFolderId]?.completedFiles || [],
failedFiles: prev[vFolderId]?.failedFiles || [],
completedBytes: prev[vFolderId]?.completedBytes || 0,
totalExpectedSize: newTotalExpectedSize,
},
};
});

startFunctions.forEach((startFunction) => {
uploadFileInfo.forEach(({ startFunction }) => {
queue.add(async () => {
let previousBytesUploaded = 0;
await startFunction({
onProgress: (bytesUploaded, bytesTotal, fileName) => {
onProgress: (bytesUploaded, _bytesTotal, fileName) => {
// Since bytesUploaded is cumulative, calculate delta from previous value
const deltaBytes = bytesUploaded - previousBytesUploaded;
previousBytesUploaded = bytesUploaded;

setUploadStatus((prev) => {
const remainingFiles = prev[vFolderId]?.pending || [];
const uploadedFilesCount =
prev[vFolderId]?.completedFiles?.length || 0;
const totalUploadedFilesCount =
(prev[vFolderId]?.completedFiles?.length || 0) +
(prev[vFolderId]?.failedFiles?.length || 0) +
(prev[vFolderId]?.pendingFiles?.length || 0);

const totalExpectedSize =
prev[vFolderId]?.totalExpectedSize || 0;
const currentCompletedBytes =
(prev[vFolderId]?.completedBytes || 0) + deltaBytes;

upsertNotification({
key: 'upload:' + vFolderId,
message: `${t('explorer.UploadToFolder', {
folderName: vFolderName,
})}${remainingFiles.length > 1 ? ` (${remainingFiles.length})` : ''}`,
})}${` (${uploadedFilesCount} / ${totalUploadedFilesCount})`}`,
backgroundTask: {
status: 'pending',
percent: Math.round((bytesUploaded / bytesTotal) * 100) - 1,
percent:
totalExpectedSize > 0
? (currentCompletedBytes / totalExpectedSize) * 100
: 0,
onChange: {
pending: t('explorer.FileInProgress', {
fileName: fileName,
Expand All @@ -119,31 +163,45 @@ const FileUploadManager: React.FC = () => {
},
});

return prev;
// handle upload progress bytes update
return {
...prev,
[vFolderId]: {
...prev[vFolderId],
completedBytes: currentCompletedBytes,
},
};
});
},
})
.then((fileName: string) => {
// handle uploaded file name only, size is already handled in progress
.then(({ name: fileName }) => {
setUploadStatus((prev) => ({
...prev,
[vFolderId]: {
...prev[vFolderId],
pending: prev[vFolderId].pending.filter(
(f) => f !== fileName,
pendingFiles: prev[vFolderId].pendingFiles.filter(
(f: string) => f !== fileName,
),
completed: [...prev[vFolderId].completed, fileName],
completedFiles: [
...(prev[vFolderId]?.completedFiles || []),
fileName,
],
},
}));
})
.catch((fileName: string) => {
.catch(({ name: fileName }) => {
setUploadStatus((prev) => ({
...prev,
[vFolderId]: {
...prev[vFolderId],
pending: prev[vFolderId].pending.filter(
(f) => f !== fileName,
pendingFiles: prev[vFolderId].pendingFiles.filter(
(f: string) => f !== fileName,
),
failed: [...prev[vFolderId].failed, fileName],
failedFiles: [
...(prev[vFolderId]?.failedFiles || []),
fileName,
],
},
}));
});
Expand All @@ -156,9 +214,9 @@ const FileUploadManager: React.FC = () => {

useEffect(() => {
Object.entries(uploadStatus).forEach(([vFolderId, status]) => {
if (!_.isEmpty(status?.pending)) return;
if (!_.isEmpty(status?.pendingFiles)) return;

if (!_.isEmpty(status?.failed)) {
if (!_.isEmpty(status?.failedFiles)) {
upsertNotification({
key: 'upload:' + vFolderId,
open: true,
Expand All @@ -174,9 +232,9 @@ const FileUploadManager: React.FC = () => {
}),
},
},
extraDescription: _.join(status?.failed, ', '),
extraDescription: _.join(status?.failedFiles, ', '),
});
} else if (!_.isEmpty(status?.completed)) {
} else if (!_.isEmpty(status?.completedFiles)) {
upsertNotification({
key: 'upload:' + vFolderId,
open: true,
Expand All @@ -196,7 +254,9 @@ const FileUploadManager: React.FC = () => {
...prev,
[vFolderId]: {
...prev[vFolderId],
completed: [],
completedFiles: [],
completedBytes: 0,
totalExpectedSize: 0,
},
}));
}
Expand All @@ -210,6 +270,8 @@ const FileUploadManager: React.FC = () => {
export default FileUploadManager;

export const useFileUploadManager = (vFolderId: string) => {
'use memo';

const baiClient = useConnectedBAIClient();
const { t } = useTranslation();
const { upsertNotification } = useSetBAINotification();
Expand Down Expand Up @@ -270,9 +332,9 @@ export const useFileUploadManager = (vFolderId: string) => {
) => {
if (!validateUploadRequest(files, vfolderId)) return;

const uploadFileNames: Array<string> = [];
const fileToUpload: Array<RcFile> = [];
const startUploadFunctionMap = _.map(files, (file) => {
uploadFileNames.push(file.webkitRelativePath || file.name);
fileToUpload.push(file);
return async (callbacks?: {
onProgress?: (
bytesUploaded: number,
Expand All @@ -289,37 +351,51 @@ export const useFileUploadManager = (vFolderId: string) => {
vfolderId,
);

return new Promise<string>((resolve, reject) => {
const upload = new tus.Upload(file, {
endpoint: uploadUrl,
uploadUrl: uploadUrl,
retryDelays: [0, 3000, 5000, 10000, 20000],
chunkSize: getOptimalChunkSize(file.size),
storeFingerprintForResuming: false, // Disable localStorage storage
metadata: {
filename: file.name,
filetype: file.type,
},
onProgress: (bytesUploaded, bytesTotal) => {
callbacks?.onProgress?.(bytesUploaded, bytesTotal, file.name);
},
onSuccess: () => {
resolve(file.webkitRelativePath || file.name);
},
onError: () => {
reject(file.webkitRelativePath || file.name);
},
});
upload.start();
});
return new Promise<{ name: string; bytes: number }>(
(resolve, reject) => {
const upload = new tus.Upload(file, {
endpoint: uploadUrl,
uploadUrl: uploadUrl,
retryDelays: [0, 3000, 5000, 10000, 20000],
chunkSize: getOptimalChunkSize(file.size),
storeFingerprintForResuming: false, // Disable localStorage storage
metadata: {
filename: file.name,
filetype: file.type,
},
onProgress: (bytesUploaded, bytesTotal) => {
callbacks?.onProgress?.(bytesUploaded, bytesTotal, file.name);
},
onSuccess: () => {
resolve({
name: file.webkitRelativePath || file.name,
bytes: file.size,
});
},
onError: () => {
reject({
name: file.webkitRelativePath || file.name,
bytes: file.size,
});
},
});
upload.start();
},
);
};
});

const uploadRequestInfo: UploadRequest = {
vFolderId: vfolderId,
vFolderName: vfolder_node?.name ?? '',
uploadFilesNames: uploadFileNames,
startFunctions: startUploadFunctionMap,
uploadFileInfo: _.zipWith(
fileToUpload,
startUploadFunctionMap,
(file, startFunction) => ({
file,
startFunction,
}),
),
};
setUploadRequests((prev) => [...prev, uploadRequestInfo]);
};
Expand Down
2 changes: 1 addition & 1 deletion react/src/components/FolderExplorerModal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ const FolderExplorerModal: React.FC<FolderExplorerProps> = ({
const bodyRef = useRef<HTMLDivElement | null>(null);

useEffect(() => {
if (uploadStatus && _.isEmpty(uploadStatus.pending)) {
if (uploadStatus && _.isEmpty(uploadStatus?.pendingFiles)) {
updateFetchKey();
}
}, [uploadStatus, updateFetchKey]);
Expand Down