From cfda632ac4f719ebcf07ec7111e1eddad61e103d Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Mon, 4 Mar 2024 16:09:23 +0100 Subject: [PATCH 1/4] Update stream-collector.ts --- .../src/stream-collector.ts | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/packages/fetch-http-handler/src/stream-collector.ts b/packages/fetch-http-handler/src/stream-collector.ts index 0e7d69cb71c..5592846a8a7 100644 --- a/packages/fetch-http-handler/src/stream-collector.ts +++ b/packages/fetch-http-handler/src/stream-collector.ts @@ -1,21 +1,14 @@ import { StreamCollector } from "@smithy/types"; -import { fromBase64 } from "@smithy/util-base64"; //reference: https://snack.expo.io/r1JCSWRGU export const streamCollector: StreamCollector = (stream: Blob | ReadableStream): Promise => { if (typeof Blob === "function" && stream instanceof Blob) { - return collectBlob(stream); + return new Uint8Array(await stream.arrayBuffer()); } return collectStream(stream as ReadableStream); }; -async function collectBlob(blob: Blob): Promise { - const base64 = await readToBase64(blob); - const arrayBuffer = fromBase64(base64); - return new Uint8Array(arrayBuffer); -} - async function collectStream(stream: ReadableStream): Promise { const chunks = []; const reader = stream.getReader(); @@ -40,26 +33,3 @@ async function collectStream(stream: ReadableStream): Promise { return collected; } - -function readToBase64(blob: Blob): Promise { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onloadend = () => { - // reference: https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL - // response from readAsDataURL is always prepended with "data:*/*;base64," - if (reader.readyState !== 2) { - return reject(new Error("Reader aborted too early")); - } - const result = (reader.result ?? "") as string; - // Response can include only 'data:' for empty blob, return empty string in this case. - // Otherwise, return the string after ',' - const commaIndex = result.indexOf(","); - const dataOffset = commaIndex > -1 ? commaIndex + 1 : result.length; - resolve(result.substring(dataOffset)); - }; - reader.onabort = () => reject(new Error("Read aborted")); - reader.onerror = () => reject(reader.error); - // reader.readAsArrayBuffer is not always available - reader.readAsDataURL(blob); - }); -} From df96295821bfe618e22f166d573d9823235cc182 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Mon, 4 Mar 2024 16:15:36 +0100 Subject: [PATCH 2/4] Update stream collector --- packages/fetch-http-handler/src/stream-collector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/fetch-http-handler/src/stream-collector.ts b/packages/fetch-http-handler/src/stream-collector.ts index 5592846a8a7..86578b22d75 100644 --- a/packages/fetch-http-handler/src/stream-collector.ts +++ b/packages/fetch-http-handler/src/stream-collector.ts @@ -1,7 +1,7 @@ import { StreamCollector } from "@smithy/types"; //reference: https://snack.expo.io/r1JCSWRGU -export const streamCollector: StreamCollector = (stream: Blob | ReadableStream): Promise => { +export const streamCollector: StreamCollector = async (stream: Blob | ReadableStream): Promise => { if (typeof Blob === "function" && stream instanceof Blob) { return new Uint8Array(await stream.arrayBuffer()); } From ca2f75c6d0fb660d74443434b096be3ab29c5c83 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Mon, 4 Mar 2024 16:18:40 +0100 Subject: [PATCH 3/4] Add changeset --- .changeset/fluffy-planes-grow.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fluffy-planes-grow.md diff --git a/.changeset/fluffy-planes-grow.md b/.changeset/fluffy-planes-grow.md new file mode 100644 index 00000000000..9a1fcefe4ff --- /dev/null +++ b/.changeset/fluffy-planes-grow.md @@ -0,0 +1,5 @@ +--- +"@smithy/fetch-http-handler": minor +--- + +Improve blob stream collector performance From 3800626621ae127d8acbd3516ea9798fd0df129b Mon Sep 17 00:00:00 2001 From: George Fu Date: Thu, 17 Oct 2024 15:09:05 +0000 Subject: [PATCH 4/4] remove FileReader where possible --- .changeset/breezy-cobras-repair.md | 7 +++ .changeset/fluffy-planes-grow.md | 5 -- .../chunked-blob-reader-native/src/index.ts | 4 ++ .../chunked-blob-reader/src/index.spec.ts | 5 ++ packages/chunked-blob-reader/src/index.ts | 37 ++++---------- .../src/stream-collector.spec.ts | 49 ++++++++----------- .../src/stream-collector.ts | 1 - 7 files changed, 46 insertions(+), 62 deletions(-) create mode 100644 .changeset/breezy-cobras-repair.md delete mode 100644 .changeset/fluffy-planes-grow.md diff --git a/.changeset/breezy-cobras-repair.md b/.changeset/breezy-cobras-repair.md new file mode 100644 index 00000000000..c2c371f4ef3 --- /dev/null +++ b/.changeset/breezy-cobras-repair.md @@ -0,0 +1,7 @@ +--- +"@smithy/chunked-blob-reader": major +"@smithy/fetch-http-handler": major +"@smithy/chunked-blob-reader-native": patch +--- + +replace FileReader with Blob.arrayBuffer() where possible diff --git a/.changeset/fluffy-planes-grow.md b/.changeset/fluffy-planes-grow.md deleted file mode 100644 index 9a1fcefe4ff..00000000000 --- a/.changeset/fluffy-planes-grow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@smithy/fetch-http-handler": minor ---- - -Improve blob stream collector performance diff --git a/packages/chunked-blob-reader-native/src/index.ts b/packages/chunked-blob-reader-native/src/index.ts index d208d5994c8..abf8db9667b 100644 --- a/packages/chunked-blob-reader-native/src/index.ts +++ b/packages/chunked-blob-reader-native/src/index.ts @@ -8,6 +8,10 @@ export function blobReader( chunkSize: number = 1024 * 1024 ): Promise { return new Promise((resolve, reject) => { + /** + * TODO(react-native): https://github.com/facebook/react-native/issues/34402 + * To drop FileReader in react-native, we need the Blob.arrayBuffer() method to work. + */ const fileReader = new FileReader(); fileReader.onerror = reject; diff --git a/packages/chunked-blob-reader/src/index.spec.ts b/packages/chunked-blob-reader/src/index.spec.ts index 290696968bb..79a5aa924ea 100644 --- a/packages/chunked-blob-reader/src/index.spec.ts +++ b/packages/chunked-blob-reader/src/index.spec.ts @@ -1,5 +1,10 @@ +import { Blob as BlobPolyfill } from "buffer"; + import { blobReader } from "./index"; +// jsdom inaccurate Blob https://github.com/jsdom/jsdom/issues/2555. +global.Blob = BlobPolyfill as any; + describe("blobReader", () => { it("reads an entire blob", async () => { const longMessage: number[] = []; diff --git a/packages/chunked-blob-reader/src/index.ts b/packages/chunked-blob-reader/src/index.ts index b3fffeaec1c..ccb312fb503 100644 --- a/packages/chunked-blob-reader/src/index.ts +++ b/packages/chunked-blob-reader/src/index.ts @@ -1,37 +1,18 @@ /** * @internal + * Reads the blob data into the onChunk consumer. */ -export function blobReader( +export async function blobReader( blob: Blob, onChunk: (chunk: Uint8Array) => void, chunkSize: number = 1024 * 1024 ): Promise { - return new Promise((resolve, reject) => { - const fileReader = new FileReader(); + const size = blob.size; + let totalBytesRead = 0; - fileReader.addEventListener("error", reject); - fileReader.addEventListener("abort", reject); - - const size = blob.size; - let totalBytesRead = 0; - - function read() { - if (totalBytesRead >= size) { - resolve(); - return; - } - fileReader.readAsArrayBuffer(blob.slice(totalBytesRead, Math.min(size, totalBytesRead + chunkSize))); - } - - fileReader.addEventListener("load", (event) => { - const result = (event.target as any).result; - onChunk(new Uint8Array(result)); - totalBytesRead += result.byteLength; - // read the next block - read(); - }); - - // kick off the read - read(); - }); + while (totalBytesRead < size) { + const slice: Blob = blob.slice(totalBytesRead, Math.min(size, totalBytesRead + chunkSize)); + onChunk(new Uint8Array(await slice.arrayBuffer())); + totalBytesRead += slice.size; + } } diff --git a/packages/fetch-http-handler/src/stream-collector.spec.ts b/packages/fetch-http-handler/src/stream-collector.spec.ts index 7f7e47e9ffc..77acc30d5b6 100644 --- a/packages/fetch-http-handler/src/stream-collector.spec.ts +++ b/packages/fetch-http-handler/src/stream-collector.spec.ts @@ -1,36 +1,29 @@ +import { Blob as BlobPolyfill } from "buffer"; + import { streamCollector } from "./stream-collector"; -/** - * Have to mock the FileReader behavior in IE, where - * reader.result is null if reads an empty blob. - */ +// jsdom inaccurate Blob https://github.com/jsdom/jsdom/issues/2555. +global.Blob = BlobPolyfill as any; + describe("streamCollector", () => { - let originalFileReader = (global as any).FileReader; - let originalBlob = (global as any).Blob; - beforeAll(() => { - originalFileReader = (global as any).FileReader; - originalBlob = (global as any).Blob; - }); - afterAll(() => { - (global as any).FileReader = originalFileReader; - (global as any).Blob = originalBlob; + const blobAvailable = typeof Blob === "function"; + const readableStreamAvailable = typeof ReadableStream === "function"; + + (blobAvailable ? it : it.skip)("collects Blob into bytearray", async () => { + const blobby = new Blob([new Uint8Array([1, 2]), new Uint8Array([3, 4])]); + const collected = await streamCollector(blobby); + expect(collected).toEqual(new Uint8Array([1, 2, 3, 4])); }); - it("returns a Uint8Array when blob is empty and when FileReader data is null(in IE)", (done) => { - (global as any).FileReader = function FileReader() { - this.result = null; //In IE, FileReader.result is null after reading empty blob - this.readAsDataURL = jest.fn().mockImplementation(() => { - if (this.onloadend) { - this.readyState = 2; - this.onloadend(); - } - }); - }; - (global as any).Blob = function Blob() {}; - const dataPromise = streamCollector(new Blob()); - dataPromise.then((data: any) => { - expect(data).toEqual(Uint8Array.from([])); - done(); + (readableStreamAvailable ? it : it.skip)("collects ReadableStream into bytearray", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2])); + controller.enqueue(new Uint8Array([3, 4])); + controller.close(); + }, }); + const collected = await streamCollector(stream); + expect(collected).toEqual(new Uint8Array([1, 2, 3, 4])); }); }); diff --git a/packages/fetch-http-handler/src/stream-collector.ts b/packages/fetch-http-handler/src/stream-collector.ts index 86578b22d75..eefa5053734 100644 --- a/packages/fetch-http-handler/src/stream-collector.ts +++ b/packages/fetch-http-handler/src/stream-collector.ts @@ -1,6 +1,5 @@ import { StreamCollector } from "@smithy/types"; -//reference: https://snack.expo.io/r1JCSWRGU export const streamCollector: StreamCollector = async (stream: Blob | ReadableStream): Promise => { if (typeof Blob === "function" && stream instanceof Blob) { return new Uint8Array(await stream.arrayBuffer());