dependencies: cache file existence when canonicalizing

After caching canonicalization, the calls to std::fs::exists were making up a
large portion of the remaining syscalls. Caching these, too, yields another
30-40% speedup.

After this commit, on large projects, the hash table for the cache itself
dominates performance measurements, which is about as far as I'm willing to go
for now.

Author: tazjin

src/dependencies/canonicalize.rs

@@ -5,7 +5,7 @@ use std::hash::Hash;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, LazyLock, RwLock};
 
-static PATH_CACHE: LazyLock<Arc<RwLock<HashMap<PathBuf, PathBuf>>>> =
+static PATH_CACHE: LazyLock<Arc<RwLock<HashMap<PathBuf, Option<PathBuf>>>>> =
     LazyLock::new(|| Default::default());
 
 /// Wrapper around [`std::fs::canonicalize`] that caches already canonicalized
@@ -14,7 +14,7 @@ static PATH_CACHE: LazyLock<Arc<RwLock<HashMap<PathBuf, PathBuf>>>> =
 /// This is done due to the operation being inherently slow: It has to walk the
 /// entire parent directory tree (especially expensive on FUSE-mounted virtual
 /// repository filesystems).
-pub fn canonicalize_cached<P>(path: P) -> Result<PathBuf, std::io::Error>
+pub fn canonicalize_cached<P>(path: P) -> Result<Option<PathBuf>, std::io::Error>
 where
     P: AsRef<Path>,
     PathBuf: Borrow<P>,
@@ -29,7 +29,12 @@ where
     }
 
     // ... then look it up ourselves.
-    let result = canonicalize(&path)?;
+    let result = if path.as_ref().exists() {
+        Some(canonicalize(&path)?)
+    } else {
+        None
+    };
+
     let mut cache = PATH_CACHE.write().unwrap();
     cache.insert(path.as_ref().to_path_buf(), result.clone());
 

src/dependencies/compiledb.rs

@@ -46,10 +46,12 @@ impl TryFrom<CompileCommandsEntry> for SourceFileEntry {
             source_file
         };
 
-        let file_path = canonicalize_cached(file_path).map_err(|source| Error::IOError {
-            source,
-            message: "canonicalize",
-        })?;
+        let file_path = canonicalize_cached(file_path)
+            .map_err(|source| Error::IOError {
+                source,
+                message: "canonicalize",
+            })?
+            .ok_or(Error::FileNotFound)?;
 
         let args = value
             .arguments
@@ -61,7 +63,7 @@ impl TryFrom<CompileCommandsEntry> for SourceFileEntry {
             .map(PathBuf::from)
             .filter_map(|p| {
                 if p.is_relative() {
-                    canonicalize_cached(start_dir.join(p)).ok()
+                    canonicalize_cached(start_dir.join(p)).ok()?
                 } else {
                     Some(p)
                 }

src/dependencies/cparse.rs

@@ -15,12 +15,7 @@ use tracing::{error, info, trace};
 /// Attempt to make the full path of head::tail
 /// returns None if that fails (e.g. path does not exist)
 fn try_resolve(head: &Path, tail: &Path) -> Option<PathBuf> {
-    let path = head.join(tail);
-    if !path.exists() {
-        return None;
-    }
-
-    canonicalize_cached(head.join(tail)).ok()
+    canonicalize_cached(head.join(tail)).ok()?
 }
 
 #[derive(PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
@@ -124,9 +119,11 @@ where
 
     for entry in paths {
         let path = match entry {
-            Ok(value) => canonicalize_cached(value).map_err(|e| Error::Internal {
-                message: format!("{:?}", e),
-            })?,
+            Ok(value) => canonicalize_cached(value)
+                .map_err(|e| Error::Internal {
+                    message: format!("{:?}", e),
+                })?
+                .ok_or(Error::FileNotFound)?,
             Err(e) => {
                 return Err(Error::Internal {
                     message: format!("{:?}", e),

src/dependencies/error.rs

@@ -29,6 +29,11 @@ pub enum Error {
     #[error("Subtask join error")]
     JoinError,
 
+    // Unfortunately, in most cases where this error can occur the path is no
+    // longer available to avoid unnecessary cloning in the hot path.
+    #[error("Required file not found")]
+    FileNotFound,
+
     #[error("Internal error")]
     Internal { message: String },
 

src/dependencies/gn.rs

@@ -30,16 +30,22 @@ pub fn load_gn_targets(
 ) -> Result<Vec<GnTarget>, Error> {
     // TODO: GN PATH?
     let mut command = Command::new("/usr/bin/gn");
-    let source_root = canonicalize_cached(source_root).map_err(|e| Error::Internal {
-        message: format!("Canonical path: {:?}", e),
-    })?;
+    let source_root = canonicalize_cached(source_root)
+        .map_err(|e| Error::Internal {
+            message: format!("Canonical path: {:?}", e),
+        })?
+        .ok_or(Error::FileNotFound)?;
 
     command.arg("desc");
     command.arg("--format=json");
     command.arg(format!("--root={}", source_root.to_string_lossy()));
-    command.arg(canonicalize_cached(gn_dir).map_err(|e| Error::Internal {
-        message: format!("Canonical path: {:?}", e),
-    })?);
+    command.arg(
+        canonicalize_cached(gn_dir)
+            .map_err(|e| Error::Internal {
+                message: format!("Canonical path: {:?}", e),
+            })?
+            .ok_or(Error::FileNotFound)?,
+    );
     command.arg(target);
     command.arg("sources");
 
@@ -93,7 +99,7 @@ pub fn load_gn_targets(
                             // otherwise assume absolute and use as-is
                             PathBuf::from(&s.as_str())
                         };
-                        canonicalize_cached(p).ok()
+                        canonicalize_cached(p).ok()?
                     })
                     .inspect(|path| {
                         info!(target: "gn-path", " - {:?}", path);