Skip to content

Commit a825191

Browse files
coyotte508coyotte508
authored
Add useXet param to all commit functions (#1711)
cc @assafvayner @jsulz @Kakulukian for viz Still not yet prod ready but allow to test more easily (including on prod with specific accountsn through the `uploadFilesWithProgress` function) One nice thing is that we don't need to call `XMLHttpRequest` anymore in `uploadFilesWithProgress`, we'll be able to remove it when 100% migrated to xet
1 parent 81cb220 commit a825191

File tree

4 files changed

+113
-91
lines changed

4 files changed

+113
-91
lines changed

‎packages/hub/src/lib/commit.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ export type CommitParams = {
9292
/**
9393
* @deprecated Not yet ready for production use
9494
*/
95-
xet?: boolean;
95+
useXet?: boolean;
9696
} & Partial<CredentialsParams>;
9797

9898
export interface CommitOutput {
@@ -298,7 +298,7 @@ export async function* commitIter(params: CommitParams): AsyncGenerator<CommitPr
298298

299299
const shaToOperation = new Map(operations.map((op, i) => [shas[i], op]));
300300

301-
if (params.xet) {
301+
if (params.useXet) {
302302
// First get all the files that are already uploaded out of the way
303303
for (const obj of json.objects) {
304304
const op = shaToOperation.get(obj.oid);

‎packages/hub/src/lib/upload-file.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ export function uploadFile(
1515
fetch?: CommitParams["fetch"];
1616
useWebWorkers?: CommitParams["useWebWorkers"];
1717
abortSignal?: CommitParams["abortSignal"];
18+
/**
19+
* @deprecated Not yet ready for production use
20+
*/
21+
useXet?: CommitParams["useXet"];
1822
} & Partial<CredentialsParams>
1923
): Promise<CommitOutput> {
2024
const path =
@@ -43,5 +47,6 @@ export function uploadFile(
4347
fetch: params.fetch,
4448
useWebWorkers: params.useWebWorkers,
4549
abortSignal: params.abortSignal,
50+
useXet: params.useXet,
4651
});
4752
}

‎packages/hub/src/lib/upload-files-with-progress.ts‎

Lines changed: 101 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ export async function* uploadFilesWithProgress(
2929
parentCommit?: CommitParams["parentCommit"];
3030
abortSignal?: CommitParams["abortSignal"];
3131
maxFolderDepth?: CommitParams["maxFolderDepth"];
32+
/**
33+
* @deprecated Not yet ready for production use
34+
*/
35+
useXet?: CommitParams["useXet"];
3236
/**
3337
* Set this to true in order to have progress events for hashing
3438
*/
@@ -51,104 +55,112 @@ export async function* uploadFilesWithProgress(
5155
parentCommit: params.parentCommit,
5256
useWebWorkers: params.useWebWorkers,
5357
abortSignal: params.abortSignal,
54-
fetch: async (input, init) => {
55-
if (!init) {
56-
return fetch(input);
57-
}
58+
useXet: params.useXet,
59+
fetch:
60+
params.useXet === true
61+
? // no need for custom fetch function if we use Xet, as we already have progress events in the commit function for file uploads in that case
62+
undefined
63+
: async (input, init) => {
64+
if (!init) {
65+
return fetch(input);
66+
}
5867

59-
if (
60-
!typedInclude(["PUT", "POST"], init.method) ||
61-
!("progressHint" in init) ||
62-
!init.progressHint ||
63-
typeof XMLHttpRequest === "undefined" ||
64-
typeof input !== "string" ||
65-
(!(init.body instanceof ArrayBuffer) &&
66-
!(init.body instanceof Blob) &&
67-
!(init.body instanceof File) &&
68-
typeof init.body !== "string")
69-
) {
70-
return fetch(input, init);
71-
}
68+
if (
69+
!typedInclude(["PUT", "POST"], init.method) ||
70+
!("progressHint" in init) ||
71+
!init.progressHint ||
72+
typeof XMLHttpRequest === "undefined" ||
73+
typeof input !== "string" ||
74+
(!(init.body instanceof ArrayBuffer) &&
75+
!(init.body instanceof Blob) &&
76+
!(init.body instanceof File) &&
77+
typeof init.body !== "string")
78+
) {
79+
return fetch(input, init);
80+
}
7281

73-
const progressHint = init.progressHint as {
74-
progressCallback: (progress: number) => void;
75-
} & (Record<string, never> | { part: number; numParts: number });
76-
const progressCallback = progressHint.progressCallback;
82+
const progressHint = init.progressHint as {
83+
progressCallback: (progress: number) => void;
84+
} & (Record<string, never> | { part: number; numParts: number });
85+
const progressCallback = progressHint.progressCallback;
7786

78-
const xhr = new XMLHttpRequest();
87+
const xhr = new XMLHttpRequest();
7988

80-
xhr.upload.addEventListener("progress", (event) => {
81-
if (event.lengthComputable) {
82-
if (progressHint.part !== undefined) {
83-
let tracking = multipartUploadTracking.get(progressCallback);
84-
if (!tracking) {
85-
tracking = { numParts: progressHint.numParts, partsProgress: {} };
86-
multipartUploadTracking.set(progressCallback, tracking);
87-
}
88-
tracking.partsProgress[progressHint.part] = event.loaded / event.total;
89-
let totalProgress = 0;
90-
for (const partProgress of Object.values(tracking.partsProgress)) {
91-
totalProgress += partProgress;
92-
}
93-
if (totalProgress === tracking.numParts) {
94-
progressCallback(0.9999999999);
95-
} else {
96-
progressCallback(totalProgress / tracking.numParts);
97-
}
98-
} else {
99-
if (event.loaded === event.total) {
100-
progressCallback(0.9999999999);
101-
} else {
102-
progressCallback(event.loaded / event.total);
103-
}
104-
}
105-
}
106-
});
89+
xhr.upload.addEventListener("progress", (event) => {
90+
if (event.lengthComputable) {
91+
if (progressHint.part !== undefined) {
92+
let tracking = multipartUploadTracking.get(progressCallback);
93+
if (!tracking) {
94+
tracking = { numParts: progressHint.numParts, partsProgress: {} };
95+
multipartUploadTracking.set(progressCallback, tracking);
96+
}
97+
tracking.partsProgress[progressHint.part] = event.loaded / event.total;
98+
let totalProgress = 0;
99+
for (const partProgress of Object.values(tracking.partsProgress)) {
100+
totalProgress += partProgress;
101+
}
102+
if (totalProgress === tracking.numParts) {
103+
progressCallback(0.9999999999);
104+
} else {
105+
progressCallback(totalProgress / tracking.numParts);
106+
}
107+
} else {
108+
if (event.loaded === event.total) {
109+
progressCallback(0.9999999999);
110+
} else {
111+
progressCallback(event.loaded / event.total);
112+
}
113+
}
114+
}
115+
});
107116

108-
xhr.open(init.method, input, true);
117+
xhr.open(init.method, input, true);
109118

110-
if (init.headers) {
111-
const headers = new Headers(init.headers);
112-
headers.forEach((value, key) => {
113-
xhr.setRequestHeader(key, value);
114-
});
115-
}
119+
if (init.headers) {
120+
const headers = new Headers(init.headers);
121+
headers.forEach((value, key) => {
122+
xhr.setRequestHeader(key, value);
123+
});
124+
}
116125

117-
init.signal?.throwIfAborted();
118-
xhr.send(init.body);
126+
init.signal?.throwIfAborted();
127+
xhr.send(init.body);
119128

120-
return new Promise((resolve, reject) => {
121-
xhr.addEventListener("load", () => {
122-
resolve(
123-
new Response(xhr.responseText, {
124-
status: xhr.status,
125-
statusText: xhr.statusText,
126-
headers: Object.fromEntries(
127-
xhr
128-
.getAllResponseHeaders()
129-
.trim()
130-
.split("\n")
131-
.map((header) => [header.slice(0, header.indexOf(":")), header.slice(header.indexOf(":") + 1).trim()])
132-
),
133-
})
134-
);
135-
});
136-
xhr.addEventListener("error", () => {
137-
reject(new Error(xhr.statusText));
138-
});
129+
return new Promise((resolve, reject) => {
130+
xhr.addEventListener("load", () => {
131+
resolve(
132+
new Response(xhr.responseText, {
133+
status: xhr.status,
134+
statusText: xhr.statusText,
135+
headers: Object.fromEntries(
136+
xhr
137+
.getAllResponseHeaders()
138+
.trim()
139+
.split("\n")
140+
.map((header) => [
141+
header.slice(0, header.indexOf(":")),
142+
header.slice(header.indexOf(":") + 1).trim(),
143+
])
144+
),
145+
})
146+
);
147+
});
148+
xhr.addEventListener("error", () => {
149+
reject(new Error(xhr.statusText));
150+
});
139151

140-
if (init.signal) {
141-
init.signal.addEventListener("abort", () => {
142-
xhr.abort();
152+
if (init.signal) {
153+
init.signal.addEventListener("abort", () => {
154+
xhr.abort();
143155

144-
try {
145-
init.signal?.throwIfAborted();
146-
} catch (err) {
147-
reject(err);
148-
}
149-
});
150-
}
151-
});
152-
},
156+
try {
157+
init.signal?.throwIfAborted();
158+
} catch (err) {
159+
reject(err);
160+
}
161+
});
162+
}
163+
});
164+
},
153165
});
154166
}

‎packages/hub/src/lib/upload-files.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ export function uploadFiles(
1616
useWebWorkers?: CommitParams["useWebWorkers"];
1717
maxFolderDepth?: CommitParams["maxFolderDepth"];
1818
abortSignal?: CommitParams["abortSignal"];
19+
/**
20+
* @deprecated Not yet ready for production use
21+
*/
22+
useXet?: CommitParams["useXet"];
1923
} & Partial<CredentialsParams>
2024
): Promise<CommitOutput> {
2125
return commit({
@@ -35,5 +39,6 @@ export function uploadFiles(
3539
fetch: params.fetch,
3640
useWebWorkers: params.useWebWorkers,
3741
abortSignal: params.abortSignal,
42+
useXet: params.useXet,
3843
});
3944
}

0 commit comments

Comments
 (0)