Turbopack: Lower the IO concurrency limit in CI tests (#84508)

@lukesandberg pointed out that we have test flakes from too many file descriptors open.

I think we should significantly raise these limits on the ci runners (@ijjk?), but given that we run 8 tests in parallel, it's probably good anyways for Turbopack to be less aggressive with file IO in our CI.

Author: bgw

.github/workflows/build_reusable.yml

@@ -110,6 +110,9 @@ env:
   NEXT_CI_RUNNER: ${{ inputs.runs_on_labels }}
   NEXT_TEST_PROXY_ADDRESS: ${{ inputs.overrideProxyAddress || '' }}
 
+  # defaults to 256, but we run a lot of tests in parallel, so the limit should be lower
+  NEXT_TURBOPACK_IO_CONCURRENCY: 64
+
 jobs:
   build:
     timeout-minutes: ${{ inputs.timeout_minutes }}

turbopack/crates/turbo-tasks-fs/src/lib.rs

@@ -31,6 +31,7 @@ mod watcher;
 use std::{
     borrow::Cow,
     cmp::{Ordering, min},
+    env,
     fmt::{self, Debug, Display, Formatter},
     fs::FileType,
     future::Future,
@@ -193,7 +194,20 @@ where
 }
 
 fn create_semaphore() -> tokio::sync::Semaphore {
-    tokio::sync::Semaphore::new(256)
+    // the semaphore isn't serialized, and we assume the environment variable doesn't change during
+    // runtime, so it's okay to access it in this untracked way.
+    static NEXT_TURBOPACK_IO_CONCURRENCY: LazyLock<usize> = LazyLock::new(|| {
+        env::var("NEXT_TURBOPACK_IO_CONCURRENCY")
+            .ok()
+            .filter(|val| !val.is_empty())
+            .map(|val| {
+                val.parse()
+                    .expect("NEXT_TURBOPACK_IO_CONCURRENCY must be a valid integer")
+            })
+            .filter(|val| *val != 0)
+            .unwrap_or(256)
+    });
+    tokio::sync::Semaphore::new(*NEXT_TURBOPACK_IO_CONCURRENCY)
 }
 
 #[turbo_tasks::value_trait]