Skip to content

Commit 8c90c81

Browse files
madhav-madhusoodananmadhav-madhusoodanan
committed
feat: parallelizing the execution of C++ and Rust test-files
1 parent bb674f3 commit 8c90c81

File tree

3 files changed

+132
-63
lines changed

3 files changed

+132
-63
lines changed

‎crates/intrinsic-test/src/common/compare.rs‎

Lines changed: 51 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -12,50 +12,69 @@ fn runner_command(runner: &str) -> Command {
1212
}
1313

1414
pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target: &str) -> bool {
15-
let (c, rust) = rayon::join(
16-
|| {
15+
let available_parallelism = std::thread::available_parallelism().unwrap().get();
16+
let c_outputs = (0..available_parallelism)
17+
.into_par_iter()
18+
.map(|i| {
1719
runner_command(runner)
18-
.arg("./intrinsic-test-programs")
20+
.arg(format!("./intrinsic-test-programs_{i}"))
1921
.current_dir("c_programs")
2022
.output()
21-
},
22-
|| {
23+
})
24+
.collect::<Vec<_>>();
25+
26+
let rust_outputs = (0..available_parallelism)
27+
.into_par_iter()
28+
.map(|i| {
2329
runner_command(runner)
24-
.arg(format!("./target/{target}/release/intrinsic-test-programs"))
30+
.arg(format!(
31+
"./target/{target}/release/intrinsic-test-programs_{i}"
32+
))
2533
.current_dir("rust_programs")
2634
.output()
27-
},
28-
);
29-
let (c, rust) = match (c, rust) {
30-
(Ok(c), Ok(rust)) => (c, rust),
35+
})
36+
.collect::<Vec<_>>();
37+
38+
let c_error = c_outputs.iter().filter(|elem| elem.is_err()).next();
39+
let rust_error = rust_outputs.iter().filter(|elem| elem.is_err()).next();
40+
match (c_error, rust_error) {
41+
(None, None) => (),
3142
failure => panic!("Failed to run: {failure:#?}"),
3243
};
3344

34-
if !c.status.success() {
35-
error!(
36-
"Failed to run C program.\nstdout: {stdout}\nstderr: {stderr}",
37-
stdout = std::str::from_utf8(&c.stdout).unwrap_or(""),
38-
stderr = std::str::from_utf8(&c.stderr).unwrap_or(""),
39-
);
40-
}
45+
let c_stdout = c_outputs
46+
.into_iter()
47+
.map(|c_elem| {
48+
let c = c_elem.unwrap();
49+
let c_stdout = std::str::from_utf8(&c.stdout).unwrap_or("").to_string();
50+
if !c.status.success() {
51+
error!(
52+
"Failed to run C program.\nstdout: {c_stdout}\nstderr: {stderr}",
53+
stderr = std::str::from_utf8(&c.stderr).unwrap_or(""),
54+
);
55+
}
56+
c_stdout
57+
})
58+
.collect::<String>();
4159

42-
if !rust.status.success() {
43-
error!(
44-
"Failed to run Rust program.\nstdout: {stdout}\nstderr: {stderr}",
45-
stdout = std::str::from_utf8(&rust.stdout).unwrap_or(""),
46-
stderr = std::str::from_utf8(&rust.stderr).unwrap_or(""),
47-
);
48-
}
60+
let rust_stdout = rust_outputs
61+
.into_iter()
62+
.map(|rust_elem| {
63+
let rust = rust_elem.unwrap();
64+
let rust_stdout = std::str::from_utf8(&rust.stdout).unwrap_or("").to_string();
65+
if !rust.status.success() {
66+
error!(
67+
"Failed to run Rust program.\nstdout: {rust_stdout}\nstderr: {stderr}",
68+
stderr = std::str::from_utf8(&rust.stderr).unwrap_or(""),
69+
);
70+
}
71+
rust_stdout
72+
})
73+
.collect::<String>();
4974

5075
info!("Completed running C++ and Rust test binaries");
51-
let c = std::str::from_utf8(&c.stdout)
52-
.unwrap()
53-
.to_lowercase()
54-
.replace("-nan", "nan");
55-
let rust = std::str::from_utf8(&rust.stdout)
56-
.unwrap()
57-
.to_lowercase()
58-
.replace("-nan", "nan");
76+
let c = c_stdout.to_lowercase().replace("-nan", "nan");
77+
let rust = rust_stdout.to_lowercase().replace("-nan", "nan");
5978

6079
let c_output_map = c
6180
.split(INTRINSIC_DELIMITER)

‎crates/intrinsic-test/src/common/gen_rust.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ fn write_cargo_toml_header(w: &mut impl std::io::Write, name: &str) -> std::io::
3939
pub fn write_bin_cargo_toml(
4040
w: &mut impl std::io::Write,
4141
module_count: usize,
42+
binary_count: usize,
4243
) -> std::io::Result<()> {
4344
write_cargo_toml_header(w, "intrinsic-test-programs")?;
4445

@@ -49,6 +50,12 @@ pub fn write_bin_cargo_toml(
4950
writeln!(w, "mod_{i} = {{ path = \"mod_{i}/\" }}")?;
5051
}
5152

53+
for i in 0..binary_count {
54+
writeln!(w, "[[bin]]")?;
55+
writeln!(w, "name = \"client\"")?;
56+
writeln!(w, "path = \"src/main_{i}.rs\"")?;
57+
}
58+
5259
Ok(())
5360
}
5461

‎crates/intrinsic-test/src/common/mod.rs‎

Lines changed: 74 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ pub mod values;
2929

3030
/// Architectures must support this trait
3131
/// to be successfully tested.
32-
pub trait SupportedArchitectureTest {
32+
pub trait SupportedArchitectureTest
33+
where
34+
Self: Sync + Send,
35+
{
3336
type IntrinsicImpl: IntrinsicTypeDefinition + Sync;
3437

3538
fn cli_options(&self) -> &ProcessedCli;
@@ -104,46 +107,86 @@ pub trait SupportedArchitectureTest {
104107
)
105108
.unwrap();
106109

107-
// This is done because `cpp_compiler_wrapped` is None when
108-
// the --generate-only flag is passed
110+
let available_parallelism = std::thread::available_parallelism().unwrap().get();
111+
self.intrinsics()
112+
.par_chunks(available_parallelism)
113+
.enumerate()
114+
.map(|(i, chunk)| {
115+
let mut file = File::create(format!("c_programs/main_{i}.cpp")).unwrap();
116+
write_main_cpp(
117+
&mut file,
118+
Self::PLATFORM_C_DEFINITIONS,
119+
Self::PLATFORM_C_HEADERS,
120+
chunk.iter().map(|i| i.name.as_str()),
121+
)
122+
})
123+
.collect::<Result<(), std::io::Error>>()
124+
.unwrap();
125+
109126
if let Some(cpp_compiler) = cpp_compiler_wrapped.as_ref() {
110-
// compile this cpp file into a .o file
111-
trace!("compiling main.cpp");
112-
let output = cpp_compiler
113-
.compile_object_file("main.cpp", "intrinsic-test-programs.o")
114-
.unwrap();
115-
assert!(output.status.success(), "{output:?}");
116-
117-
let object_files = (0..chunk_count)
118-
.map(|i| format!("mod_{i}.o"))
119-
.chain(["intrinsic-test-programs.o".to_owned()]);
120-
121-
let output = cpp_compiler
122-
.link_executable(object_files, "intrinsic-test-programs")
123-
.unwrap();
124-
assert!(output.status.success(), "{output:?}");
125-
}
127+
(0..available_parallelism)
128+
.into_par_iter()
129+
.map(|i| {
130+
// This is done because `cpp_compiler_wrapped` is None when
131+
// the --generate-only flag is passed
132+
// compile this cpp file into a .o file
133+
trace!("compiling intrinsic-test-programs_{i}.cpp");
134+
let output = cpp_compiler.compile_object_file(
135+
format!("main_{i}.cpp").as_str(),
136+
format!("main_{i}.o").as_str(),
137+
);
138+
139+
if output.is_err() {
140+
return output;
141+
};
142+
143+
let object_files = (0..chunk_count)
144+
.map(|mod_i| format!("mod_{mod_i}.o"))
145+
.chain([format!("main_{i}.o").to_owned()]);
146+
147+
let output = cpp_compiler.link_executable(
148+
object_files,
149+
format!("intrinsic-test-programs_{i}").as_str(),
150+
);
126151

127-
true
152+
return output;
153+
})
154+
.inspect(|output| {
155+
assert!(output.is_ok(), "{output:?}");
156+
if let Ok(out) = &output {
157+
assert!(out.status.success(), "{output:?}")
158+
}
159+
})
160+
.any(|output| output.is_err())
161+
} else {
162+
true
163+
}
128164
}
129165

130166
fn build_rust_file(&self) -> bool {
131167
std::fs::create_dir_all("rust_programs/src").unwrap();
132168

133169
let (chunk_size, chunk_count) = manual_chunk(self.intrinsics().len(), 400);
170+
let available_parallelism = std::thread::available_parallelism().unwrap().get();
134171

135172
let mut cargo = File::create("rust_programs/Cargo.toml").unwrap();
136-
write_bin_cargo_toml(&mut cargo, chunk_count).unwrap();
137-
138-
let mut main_rs = File::create("rust_programs/src/main.rs").unwrap();
139-
write_main_rs(
140-
&mut main_rs,
141-
chunk_count,
142-
Self::PLATFORM_RUST_CFGS,
143-
"",
144-
self.intrinsics().iter().map(|i| i.name.as_str()),
145-
)
146-
.unwrap();
173+
write_bin_cargo_toml(&mut cargo, chunk_count, available_parallelism).unwrap();
174+
175+
self.intrinsics()
176+
.par_chunks(available_parallelism)
177+
.enumerate()
178+
.map(|(i, chunk)| {
179+
let mut main_rs = File::create(format!("rust_programs/src/main_{i}.rs")).unwrap();
180+
write_main_rs(
181+
&mut main_rs,
182+
chunk_count,
183+
Self::PLATFORM_RUST_CFGS,
184+
"",
185+
chunk.iter().map(|i| i.name.as_str()),
186+
)
187+
})
188+
.collect::<Result<(), std::io::Error>>()
189+
.unwrap();
147190

148191
let target = &self.cli_options().target;
149192
let toolchain = self.cli_options().toolchain.as_deref();

0 commit comments

Comments
 (0)