ref: Run upload preparation with maximum concurrency (#382)

Author: Luca Forstner

CHANGELOG.md

@@ -4,6 +4,10 @@
 
 - "You know what they say ‘Fool me once, strike one, but fool me twice… strike three.’" — Michael Scott
 
+## 0.6.1
+
+- ref: Run upload preparation with maximum concurrency (#382)
+
 ## 0.6.0
 
 - feat(webpack): Add debug ID injection to the webpack plugin (#198)

packages/bundler-plugin-core/src/debug-id.ts

@@ -67,9 +67,9 @@ export async function prepareBundleForDebugIdUpload(
     bundleFilePath,
     bundleContent,
     logger
-  ).then(async (sourceMapPath): Promise<void> => {
+  ).then(async (sourceMapPath) => {
     if (sourceMapPath) {
-      return await prepareSourceMapForDebugIdUpload(
+      await prepareSourceMapForDebugIdUpload(
         sourceMapPath,
         path.join(uploadFolder, `${uniqueUploadName}.js.map`),
         debugId,
@@ -78,7 +78,8 @@ export async function prepareBundleForDebugIdUpload(
     }
   });
 
-  return Promise.all([writeSourceFilePromise, writeSourceMapFilePromise]);
+  await writeSourceFilePromise;
+  await writeSourceMapFilePromise;
 }
 
 /**

packages/bundler-plugin-core/src/index.ts

@@ -321,16 +321,31 @@ const unplugin = createUnplugin<Options>((options, unpluginMetaContext) => {
               path.join(os.tmpdir(), "sentry-bundler-plugin-upload-")
             );
 
-            await Promise.all(
-              debugIdChunkFilePaths.map(async (chunkFilePath, chunkIndex): Promise<void> => {
+            // Preparing the bundles can be a lot of work and doing it all at once has the potential of nuking the heap so
+            // instead we do it with a maximum of 16 concurrent workers
+            const preparationTasks = debugIdChunkFilePaths.map(
+              (chunkFilePath, chunkIndex) => async () => {
                 await prepareBundleForDebugIdUpload(
                   chunkFilePath,
                   await sourceFileUploadFolderPromise,
                   String(chunkIndex),
                   logger
                 );
-              })
+              }
             );
+            const workers: Promise<void>[] = [];
+            const worker = async () => {
+              while (preparationTasks.length > 0) {
+                const task = preparationTasks.shift();
+                if (task) {
+                  await task();
+                }
+              }
+            };
+            for (let workerIndex = 0; workerIndex < 16; workerIndex++) {
+              workers.push(worker());
+            }
+            await Promise.all(workers);
 
             tmpUploadFolder = await sourceFileUploadFolderPromise;
             await uploadDebugIdSourcemaps(internalOptions, ctx, tmpUploadFolder, releaseName);