feat(build): Added dynamic resource limit handling

This commit adds handling to attempt to increase the max file
descriptors if we detect that there is a risk of hitting the limit.

If we cannot increase the max file descriptors, we fall back to coarse
grain locking as that is better than a build crashing due to resource
limits.

Author: ranger-ross

src/cargo/core/compiler/build_runner/mod.rs

@@ -10,12 +10,14 @@ use crate::core::compiler::locking::LockingMode;
 use crate::core::compiler::{self, Unit, artifact};
 use crate::util::cache_lock::CacheLockMode;
 use crate::util::errors::CargoResult;
+use crate::util::rlimit;
 use annotate_snippets::{Level, Message};
 use anyhow::{Context as _, bail};
 use cargo_util::paths;
 use filetime::FileTime;
 use itertools::Itertools;
 use jobserver::Client;
+use tracing::{debug, warn};
 
 use super::build_plan::BuildPlan;
 use super::custom_build::{self, BuildDeps, BuildScriptOutputs, BuildScripts};
@@ -117,7 +119,7 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         };
 
         let locking_mode = if bcx.gctx.cli_unstable().fine_grain_locking {
-            LockingMode::Fine
+            determine_locking_mode(&bcx)?
         } else {
             LockingMode::Coarse
         };
@@ -736,3 +738,49 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         }
     }
 }
+
+/// Determines if we have enough resources to safely use fine grain locking.
+/// This function will raises the number of max number file descriptors the current process
+/// can have open at once to make sure we are able to compile without running out of fds.
+///
+/// If we cannot acquire a safe max number of file descriptors, we fallback to coarse grain
+/// locking.
+pub fn determine_locking_mode(bcx: &BuildContext<'_, '_>) -> CargoResult<LockingMode> {
+    let total_units = bcx.unit_graph.keys().len() as u64;
+
+    // This is a bit arbitrary but if we do not have at least 10 times the remaining file
+    // descriptors than total build units there is a chance we could hit the limit.
+    // This is a fairly conservative estimate to make sure we don't hit max fd errors.
+    let safe_threshold = total_units * 10;
+
+    let Ok(mut limit) = rlimit::get_max_file_descriptors() else {
+        return Ok(LockingMode::Coarse);
+    };
+
+    if limit.soft_limit >= safe_threshold {
+        // The limit is higher or equal to what we deemed safe, so
+        // there is no need to raise the limit.
+        return Ok(LockingMode::Fine);
+    }
+
+    let display_fd_warning = || -> CargoResult<()> {
+        bcx.gctx.shell().verbose(|shell| shell.warn("ulimit was to low to safely enable fine grain locking, falling back to coarse grain locking"))
+    };
+
+    if limit.hard_limit < safe_threshold {
+        // The max we could raise the limit to is still not enough to safely compile.
+        display_fd_warning()?;
+        return Ok(LockingMode::Coarse);
+    }
+
+    limit.soft_limit = safe_threshold;
+
+    debug!("raising fd limit to {safe_threshold}");
+    if let Err(err) = rlimit::set_max_file_descriptors(limit) {
+        warn!("failed to raise max fds: {err}");
+        display_fd_warning()?;
+        return Ok(LockingMode::Coarse);
+    }
+
+    return Ok(LockingMode::Fine);
+}

src/cargo/util/mod.rs

@@ -61,6 +61,7 @@ mod once;
 mod progress;
 mod queue;
 pub mod restricted_names;
+pub mod rlimit;
 pub mod rustc;
 mod semver_eval_ext;
 mod semver_ext;

src/cargo/util/rlimit.rs

@@ -0,0 +1,106 @@
+#[cfg(unix)]
+use libc::{RLIMIT_NOFILE, getrlimit, rlimit, setrlimit};
+
+use crate::CargoResult;
+
+pub struct ResourceLimits {
+    pub soft_limit: u64,
+    pub hard_limit: u64,
+}
+
+#[cfg(unix)]
+pub fn get_max_file_descriptors() -> CargoResult<ResourceLimits> {
+    let mut rlim = rlimit {
+        rlim_cur: 0,
+        rlim_max: 0,
+    };
+
+    let result = unsafe { getrlimit(RLIMIT_NOFILE, &mut rlim) };
+    if result != 0 {
+        return Err(anyhow::Error::from(std::io::Error::last_os_error())
+            .context("Failed to get rlimit with error code"));
+    }
+
+    return Ok(ResourceLimits {
+        soft_limit: rlim.rlim_cur,
+        hard_limit: rlim.rlim_max,
+    });
+}
+
+#[cfg(windows)]
+pub fn get_max_file_descriptors() -> CargoResult<ResourceLimits> {
+    let soft_limit = windows::getmaxstdio() as u64;
+
+    return Ok(ResourceLimits {
+        soft_limit,
+        // Windows does not provide a way to get the hard limit so we return a estimated max.
+        // This is likely less than max for some systems but should be supported by most and is
+        // likely high enough that most projects do not run out of file descriptors.
+        // See: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-170#remarks
+        hard_limit: 8192,
+    });
+}
+
+#[cfg(unix)]
+pub fn set_max_file_descriptors(limits: ResourceLimits) -> CargoResult<()> {
+    let rlim = rlimit {
+        rlim_cur: limits.soft_limit,
+        rlim_max: limits.hard_limit,
+    };
+    let result = unsafe { setrlimit(RLIMIT_NOFILE, &rlim) };
+    if result != 0 {
+        return Err(anyhow::Error::from(std::io::Error::last_os_error())
+            .context("Failed to set rlimit with error code"));
+    }
+
+    return Ok(());
+}
+
+#[cfg(windows)]
+pub fn set_max_file_descriptors(limits: ResourceLimits) -> CargoResult<()> {
+    windows::setmaxstdio(limits.soft_limit as u32)?;
+    return Ok(());
+}
+
+#[cfg(windows)]
+mod windows {
+    use std::io;
+    use std::os::raw::c_int;
+
+    unsafe extern "C" {
+        fn _setmaxstdio(new_max: c_int) -> c_int;
+        fn _getmaxstdio() -> c_int;
+    }
+
+    /// Sets a maximum for the number of simultaneously open files at the stream I/O level.
+    ///
+    /// See <https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-170>
+    pub fn setmaxstdio(new_max: u32) -> io::Result<u32> {
+        // A negative `new_max` will cause EINVAL.
+        // A negative `ret` should never appear.
+        // It is safe even if the return value is wrong.
+        #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
+        unsafe {
+            let ret = _setmaxstdio(new_max as c_int);
+            if ret < 0 {
+                return Err(io::Error::last_os_error());
+            }
+            Ok(ret as u32)
+        }
+    }
+
+    /// Returns the number of simultaneously open files permitted at the stream I/O level.
+    ///
+    /// See <https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/getmaxstdio?view=msvc-170>
+    #[must_use]
+    pub fn getmaxstdio() -> u32 {
+        // A negative `ret` should never appear.
+        // It is safe even if the return value is wrong.
+        #[allow(clippy::cast_sign_loss)]
+        unsafe {
+            let ret = _getmaxstdio();
+            debug_assert!(ret >= 0);
+            ret as u32
+        }
+    }
+}