Rollup merge of rust-lang#145408 - Kobzol:deduplicate-search-paths, r=petrochenkov

Deduplicate -L search paths

For each -L passed to the compiler, we eagerly scan the whole directory. If it has a lot of files, that results in a lot of allocations. So it's needless to do this if some -L paths are actually duplicated (which can happen e.g. in the situation in the linked issue).

This PR both deduplicates the args, and also teaches rustdoc not to pass duplicated args to merged doctests.

Fixes: rust-lang#145375

Author: Kobzol

compiler/rustc_session/src/config.rs

@@ -2847,16 +2847,27 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
         // This is the location used by the `rustc-dev` `rustup` component.
         real_source_base_dir("lib/rustlib/rustc-src/rust", "compiler/rustc/src/main.rs");
 
-    let mut search_paths = vec![];
-    for s in &matches.opt_strs("L") {
-        search_paths.push(SearchPath::from_cli_opt(
-            sysroot.path(),
-            &target_triple,
-            early_dcx,
-            s,
-            unstable_opts.unstable_options,
-        ));
-    }
+    // We eagerly scan all files in each passed -L path. If the same directory is passed multiple
+    // times, and the directory contains a lot of files, this can take a lot of time.
+    // So we remove -L paths that were passed multiple times, and keep only the first occurrence.
+    // We still have to keep the original order of the -L arguments.
+    let search_paths: Vec<SearchPath> = {
+        let mut seen_search_paths = FxHashSet::default();
+        let search_path_matches: Vec<String> = matches.opt_strs("L");
+        search_path_matches
+            .iter()
+            .filter(|p| seen_search_paths.insert(*p))
+            .map(|path| {
+                SearchPath::from_cli_opt(
+                    sysroot.path(),
+                    &target_triple,
+                    early_dcx,
+                    &path,
+                    unstable_opts.unstable_options,
+                )
+            })
+            .collect()
+    };
 
     let working_dir = std::env::current_dir().unwrap_or_else(|e| {
         early_dcx.early_fatal(format!("Current directory is invalid: {e}"));

src/librustdoc/doctest.rs

@@ -16,7 +16,7 @@ use std::{fmt, panic, str};
 
 pub(crate) use make::{BuildDocTestBuilder, DocTestBuilder};
 pub(crate) use markdown::test as test_markdown;
-use rustc_data_structures::fx::{FxHashMap, FxHasher, FxIndexMap, FxIndexSet};
+use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxHasher, FxIndexMap, FxIndexSet};
 use rustc_errors::emitter::HumanReadableErrorType;
 use rustc_errors::{ColorConfig, DiagCtxtHandle};
 use rustc_hir as hir;
@@ -689,6 +689,10 @@ fn run_test(
             "--extern=doctest_bundle_{edition}=",
             edition = doctest.edition
         ));
+
+        // Deduplicate passed -L directory paths, since usually all dependencies will be in the
+        // same directory (e.g. target/debug/deps from Cargo).
+        let mut seen_search_dirs = FxHashSet::default();
         for extern_str in &rustdoc_options.extern_strs {
             if let Some((_cratename, path)) = extern_str.split_once('=') {
                 // Direct dependencies of the tests themselves are
@@ -698,7 +702,9 @@ fn run_test(
                     .parent()
                     .filter(|x| x.components().count() > 0)
                     .unwrap_or(Path::new("."));
-                runner_compiler.arg("-L").arg(dir);
+                if seen_search_dirs.insert(dir) {
+                    runner_compiler.arg("-L").arg(dir);
+                }
             }
         }
         let output_bundle_file = doctest