difftest: move lib into tests workspace, move mod config to new crate types, massive difftest speedup 36s -> 6s

Author: Firestar99

.config/nextest.toml

@@ -6,4 +6,4 @@ fail-fast = false
 default-filter = 'package(difftests)'
 
 [profile.difftest-runner]
-default-filter = 'package(difftest-runner) | package(difftest)'
+default-filter = 'package(difftest-runner) | package(difftest-types)'

.github/workflows/ci.yaml

@@ -190,7 +190,7 @@ jobs:
       - name: cargo fetch --locked difftests
         run: cargo fetch --locked --manifest-path=tests/difftests/tests/Cargo.toml --target $TARGET
       - name: test difftest-runner
-        run: cargo nextest run -P difftest-runner -p difftest-runner -p difftest --release --no-default-features --features "use-installed-tools"
+        run: cargo nextest run -P difftest-runner -p difftest-runner -p difftest-types --release --no-default-features --features "use-installed-tools"
       - name: difftests
         run: cargo nextest run -P difftests -p difftests --release --no-default-features --features "use-installed-tools"
 

.github/workflows/lint.sh

@@ -49,8 +49,10 @@ clippy_no_features examples/shaders/reduce
 
 # tests
 clippy tests/compiletests
+clippy_no_features tests/difftests/types
+clippy tests/difftests/runner
 clippy tests/difftests/bin
-clippy tests/difftests/lib
+clippy tests/difftests/tests/lib
 
 # Custom lints
 

Cargo.lock

@@ -29,7 +29,7 @@ members = [
     "tests/compiletests/deps-helper",
     "tests/difftests/bin",
     "tests/difftests/runner",
-    "tests/difftests/lib",
+    "tests/difftests/types",
 ]
 
 [workspace.package]

Cargo.toml

@@ -23,7 +23,7 @@ thiserror = "1.0"
 toml = { version = "0.8.20", default-features = false, features = ["parse"] }
 bytesize = "2.0.1"
 bytemuck = "1.21.0"
-difftest = { path = "../lib" }
+difftest-types = { path = "../types" }
 tabled = { version = "0.15", default-features = false, features = ["std"] }
 
 [dev-dependencies]

tests/difftests/runner/Cargo.toml

@@ -1,6 +1,6 @@
 #![allow(clippy::unimplemented)]
 
-use difftest::config::OutputType;
+use difftest_types::config::OutputType;
 use std::marker::PhantomData;
 
 /// Represents the magnitude of a difference between two values

tests/difftests/runner/src/differ.rs

@@ -1,6 +1,7 @@
 use bytesize::ByteSize;
-use difftest::config::{OutputType, TestMetadata};
+use difftest_types::config::{OutputType, TestMetadata};
 use serde::{Deserialize, Serialize};
+use std::process::Stdio;
 use std::{
     collections::{HashMap, HashSet},
     fs,
@@ -288,6 +289,8 @@ impl Runner {
 
             let output = cmd
                 .current_dir(&package.absolute_path)
+                .stdout(Stdio::inherit())
+                .stderr(Stdio::inherit())
                 .output()
                 .map_err(|e| RunnerError::Io { source: e })?;
             let exit_code = output.status.code().unwrap_or(-1);
@@ -651,7 +654,7 @@ pub fn forward_features(cmd: &mut Command) {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use difftest::config::OutputType;
+    use difftest_types::config::OutputType;
     use std::{fs, io::Write, path::Path, path::PathBuf};
     use tempfile::{NamedTempFile, tempdir};
 
@@ -862,8 +865,7 @@ mod tests {
     fn test_invalid_metadata_json() {
         // Test that invalid JSON in metadata file causes proper error
         let metadata_content = "{ invalid json }";
-        let result: Result<difftest::config::TestMetadata, _> =
-            serde_json::from_str(metadata_content);
+        let result: Result<TestMetadata, _> = serde_json::from_str(metadata_content);
         assert!(result.is_err());
         // Just check that it's an error, don't check the specific message
     }

tests/difftests/runner/src/runner.rs

@@ -1,4 +1,5 @@
 use crate::runner::RunnerResult;
+use std::ffi::OsStr;
 use std::fmt::{Display, Formatter};
 use std::fs;
 use std::path::{Path, PathBuf};
@@ -47,6 +48,14 @@ impl TestCase {
         for entry in fs::read_dir(&self.absolute_path)? {
             let entry = entry?;
             let path = entry.path();
+            let relative_path = self.relative_path.join(entry.file_name());
+            if IGNORE_DIR_LIST
+                .iter()
+                .any(|dir| relative_path.as_os_str() == OsStr::new(dir))
+            {
+                debug!("Ignoring test binary: {}", path.display());
+                return Ok(());
+            }
             if path.is_dir() && path.join("Cargo.toml").exists() {
                 debug!("Found binary package candidate: {}", path.display());
                 self.test_binaries
@@ -93,11 +102,17 @@ impl Display for TestBinary {
     }
 }
 
+/// List of paths relative to `./tests/difftests/tests/` to ignore
+pub const IGNORE_DIR_LIST: &[&str] = &["target", "lib"];
+
 pub fn collect_test_dirs(root: &Path) -> RunnerResult<Vec<TestCase>> {
     fn recurse(root: &Path, traverse: &Path, test_cases: &mut Vec<TestCase>) -> RunnerResult<()> {
         let absolute_path = root.join(traverse);
-        // skip target dir
-        if absolute_path.file_name() == Some(std::ffi::OsStr::new("target")) {
+        if IGNORE_DIR_LIST
+            .iter()
+            .any(|dir| traverse.as_os_str() == OsStr::new(dir))
+        {
+            debug!("Ignoring path: {}", absolute_path.display());
             return Ok(());
         }