Use different hashing functions for different file types (#1135)

* Use different hashing functions for different file types

This adds a new special hasher for static libraries, which contain timestamps
and other info that prevent them from being cachable in some cases.

* Update src/util.rs

Co-authored-by: Mitchell Hentges <[email protected]>

Co-authored-by: Jack Moffitt <[email protected]>
Co-authored-by: Mitchell Hentges <[email protected]>

Author: 3 people

Cargo.toml

@@ -23,7 +23,7 @@ required-features = ["dist-server"]
 
 [dependencies]
 anyhow = "1.0"
-ar = { version = "0.9", optional = true }
+ar = "0.9"
 async-trait = "0.1"
 atty = "0.2.6"
 base64 = "0.13"
@@ -129,7 +129,7 @@ native-zlib = []
 # Enable features that require unstable features of Nightly Rust.
 unstable = []
 # Enables distributed support in the sccache client
-dist-client = ["ar", "flate2", "hyper", "hyperx", "reqwest", "url", "sha2"]
+dist-client = ["flate2", "hyper", "hyperx", "reqwest", "url", "sha2"]
 # Enables the sccache-dist binary
 dist-server = ["crossbeam-utils", "jsonwebtoken", "flate2", "hyperx", "libmount", "nix", "openssl", "reqwest", "rouille", "syslog", "void", "version-compare"]
 # Enables dist tests with external requirements

src/compiler/rust.rs

@@ -25,7 +25,7 @@ use crate::dist::pkg;
 #[cfg(feature = "dist-client")]
 use crate::lru_disk_cache::{LruCache, Meter};
 use crate::mock_command::{CommandCreatorSync, RunCommand};
-use crate::util::{fmt_duration_as_secs, hash_all, run_input_output, Digest};
+use crate::util::{fmt_duration_as_secs, hash_all, hash_all_archives, run_input_output, Digest};
 use crate::util::{ref_env, HashToDigest, OsStrExt};
 use filetime::FileTime;
 use log::Level::Trace;
@@ -1337,7 +1337,7 @@ where
         // Hash the contents of the staticlibs listed on the commandline.
         trace!("[{}]: hashing {} staticlibs", crate_name, staticlibs.len());
         let abs_staticlibs = staticlibs.iter().map(|s| cwd.join(s)).collect::<Vec<_>>();
-        let staticlib_hashes = hash_all(&abs_staticlibs, pool);
+        let staticlib_hashes = hash_all_archives(&abs_staticlibs, pool);
 
         let ((source_files, source_hashes, mut env_deps), extern_hashes, staticlib_hashes) = futures::try_join!(
             source_files_and_hashes_and_env_deps,
@@ -2343,7 +2343,7 @@ mod test {
     use itertools::Itertools;
     use std::ffi::OsStr;
     use std::fs::File;
-    use std::io::Write;
+    use std::io::{self, Write};
     use std::sync::{Arc, Mutex};
 
     fn _parse_arguments(arguments: &[String]) -> CompilerArguments<ParsedArguments> {
@@ -2953,13 +2953,28 @@ c:/foo/bar.rs:
 
     #[test]
     fn test_generate_hash_key() {
+        use ar::{Builder, Header};
         drop(env_logger::try_init());
         let f = TestFixture::new();
         const FAKE_DIGEST: &str = "abcd1234";
+        const BAZ_O_SIZE: u64 = 1024;
         // We'll just use empty files for each of these.
-        for s in ["foo.rs", "bar.rs", "bar.rlib", "libbaz.a"].iter() {
+        for s in ["foo.rs", "bar.rs", "bar.rlib"].iter() {
             f.touch(s).unwrap();
         }
+        // libbaz.a needs to be a valid archive.
+        create_file(f.tempdir.path(), "libbaz.a", |f| {
+            let mut builder = Builder::new(f);
+            let hdr = Header::new(b"baz.o".to_vec(), BAZ_O_SIZE);
+            builder.append(&hdr, io::repeat(0).take(BAZ_O_SIZE))?;
+            Ok(())
+        })
+        .unwrap();
+        let mut m = Digest::new();
+        m.update(b"baz.o");
+        m.update(&vec![0; BAZ_O_SIZE as usize]);
+        let libbaz_a_digest = m.finish();
+
         let mut emit = HashSet::new();
         emit.insert("link".to_string());
         emit.insert("metadata".to_string());
@@ -3038,8 +3053,9 @@ c:/foo/bar.rs:
         m.update(empty_digest.as_bytes());
         // bar.rlib (extern crate, from externs)
         m.update(empty_digest.as_bytes());
-        // libbaz.a (static library, from staticlibs)
-        m.update(empty_digest.as_bytes());
+        // libbaz.a (static library, from staticlibs), containing a single
+        // file, baz.o, consisting of 1024 bytes of zeroes.
+        m.update(libbaz_a_digest.as_bytes());
         // Env vars
         OsStr::new("CARGO_BLAH").hash(&mut HashToDigest { digest: &mut m });
         m.update(b"=");

src/util.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::mock_command::{CommandChild, RunCommand};
+use ar::Archive;
 use blake3::Hasher as blake3_Hasher;
 use byteorder::{BigEndian, ByteOrder};
 use serde::Serialize;
@@ -22,6 +23,7 @@ use std::hash::Hasher;
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
 use std::process::{self, Stdio};
+use std::str;
 use std::time;
 use std::time::Duration;
 
@@ -121,6 +123,61 @@ pub async fn hash_all(files: &[PathBuf], pool: &tokio::runtime::Handle) -> Resul
     Ok(hashes)
 }
 
+/// Calculate the digest of each static library archive in `files` on background threads in
+/// `pool`.
+///
+/// The hash is calculated by adding the filename of each archive entry followed
+/// by its contents, ignoring headers and other file metadata. This primarily
+/// exists because Apple's `ar` tool inserts timestamps for each file with
+/// no way to disable this behavior.
+pub async fn hash_all_archives(
+    files: &[PathBuf],
+    pool: &tokio::runtime::Handle,
+) -> Result<Vec<String>> {
+    let start = time::Instant::now();
+    let count = files.len();
+    let iter = files.iter().map(|path| {
+        let path = path.clone();
+        pool.spawn_blocking(move || -> Result<String> {
+            let mut m = Digest::new();
+            let reader = File::open(&path)
+                .with_context(|| format!("Failed to open file for hashing: {:?}", path))?;
+            let mut archive = Archive::new(reader);
+            while let Some(entry) = archive.next_entry() {
+                let entry = entry?;
+                m.update(entry.header().identifier());
+                update_from_reader(&mut m, entry)?;
+            }
+            Ok(m.finish())
+        })
+    });
+
+    let mut hashes = futures::future::try_join_all(iter).await?;
+    if let Some(i) = hashes.iter().position(|res| res.is_err()) {
+        return Err(hashes.swap_remove(i).unwrap_err());
+    }
+
+    trace!(
+        "Hashed {} files in {}",
+        count,
+        fmt_duration_as_secs(&start.elapsed())
+    );
+    Ok(hashes.into_iter().map(|res| res.unwrap()).collect())
+}
+
+/// Update the digest `m` with all data from `reader`.
+fn update_from_reader<R: Read>(m: &mut Digest, mut reader: R) -> Result<()> {
+    loop {
+        let mut buffer = [0; 1024];
+        let count = reader.read(&mut buffer[..])?;
+        if count == 0 {
+            break;
+        }
+        m.update(&buffer[..count]);
+    }
+    Ok(())
+}
+
 /// Format `duration` as seconds with a fractional component.
 pub fn fmt_duration_as_secs(duration: &Duration) -> String {
     format!("{}.{:03} s", duration.as_secs(), duration.subsec_millis())