Skip to content

Commit 118e1a4

Browse files
not-matthiasnot-matthias
committed
chore: better integration tests
1 parent 6c30d0c commit 118e1a4

File tree

33 files changed

+3012
-29
lines changed

33 files changed

+3012
-29
lines changed

‎go-runner/src/integration_tests.rs

Lines changed: 56 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
use itertools::Itertools;
12
use rstest::rstest;
2-
use std::path::PathBuf;
3+
use std::path::{Path, PathBuf};
34
use std::sync::Mutex;
45
use tempfile::TempDir;
56

@@ -17,43 +18,72 @@ fn setup_test_project(project_name: &str) -> anyhow::Result<TempDir> {
1718
Ok(temp_dir)
1819
}
1920

21+
fn assert_results_snapshots(profile_dir: &Path, project_name: &str) {
22+
let glob_pattern = profile_dir.join("results");
23+
if !glob_pattern.exists() {
24+
eprintln!("No results found for project: {project_name}");
25+
return;
26+
}
27+
28+
let files = std::fs::read_dir(&glob_pattern)
29+
.expect("Failed to read results directory")
30+
.filter_map(Result::ok)
31+
.map(|entry| entry.path())
32+
.map(|path| {
33+
let file = std::fs::File::open(&path).unwrap();
34+
serde_json::from_reader::<_, WalltimeResults>(file).unwrap()
35+
})
36+
// Ensure we have the correct order for multiple test executables
37+
.sorted_by_cached_key(|r| {
38+
r.benchmarks
39+
.iter()
40+
.map(|b| b.metadata.name.clone())
41+
.sorted()
42+
.join(";")
43+
})
44+
.collect::<Vec<_>>();
45+
46+
for (i, mut content) in files.into_iter().enumerate() {
47+
content
48+
.benchmarks
49+
.sort_by_cached_key(|b| b.metadata.name.clone());
50+
51+
let _guard = {
52+
let mut settings = insta::Settings::clone_current();
53+
settings.set_snapshot_suffix(format!("{project_name}_{i}"));
54+
settings.bind_to_scope()
55+
};
56+
57+
insta::assert_json_snapshot!(content, {
58+
".creator.pid" => "[pid]",
59+
".benchmarks[].stats" => "[stats]",
60+
});
61+
}
62+
}
63+
2064
#[rstest]
21-
// #[case::caddy("caddy")]
22-
// #[case::hugo("hugo")]
23-
// #[case::fzf("fzf")]
24-
// #[case::opentelemetry_go("opentelemetry-go")]
25-
// #[case::golang_benchmarks("golang-benchmarks")]
26-
// #[case::fuego("fuego")]
27-
// #[case::zerolog("zerolog")]
65+
// // #[case::caddy("caddy")]
66+
#[case::fzf("fzf")]
67+
#[case::opentelemetry_go("opentelemetry-go")]
68+
#[case::golang_benchmarks("golang-benchmarks")]
69+
#[case::zerolog("zerolog")]
2870
#[case::zap("zap")]
29-
// #[case::cli_runtime("cli-runtime")]
71+
#[case::hugo("hugo")]
72+
// Currently not producing results:
73+
#[case::fuego("fuego")]
74+
#[case::cli_runtime("cli-runtime")]
3075
fn test_build_and_run(#[case] project_name: &str) {
3176
let temp_dir = setup_test_project(project_name).unwrap();
3277

3378
// Mutex to prevent concurrent tests from interfering with CODSPEED_PROFILE_FOLDER env var
3479
static ENV_MUTEX: Mutex<()> = Mutex::new(());
35-
let _guard = ENV_MUTEX.lock().unwrap();
80+
let _env_guard = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
3681

3782
let profile_dir = temp_dir.path().join("profile");
3883
unsafe { std::env::set_var("CODSPEED_PROFILE_FOLDER", &profile_dir) };
3984
if let Err(error) = crate::run_benchmarks(temp_dir.path(), ".") {
4085
panic!("Benchmarks couldn't run: {error}");
4186
}
4287

43-
let glob_pattern = profile_dir.join("results");
44-
let files = std::fs::read_dir(&glob_pattern)
45-
.expect("Failed to read results directory")
46-
.filter_map(Result::ok)
47-
.map(|entry| entry.path())
48-
.collect::<Vec<_>>();
49-
assert_eq!(files.len(), 1);
50-
51-
let results_file = files.first().unwrap();
52-
let content = std::fs::read_to_string(results_file).unwrap();
53-
let content = serde_json::from_str::<WalltimeResults>(&content).unwrap();
54-
55-
insta::assert_json_snapshot!(content, {
56-
".creator.pid" => "[pid]",
57-
".benchmarks[].stats" => "[stats]",
58-
});
88+
assert_results_snapshots(&profile_dir, project_name);
5989
}

‎go-runner/src/lib.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,12 @@ pub fn run_benchmarks(project_dir: &Path, bench: &str) -> anyhow::Result<()> {
6262
for bench_path in &binaries {
6363
info!("Running: {bench_path:?}");
6464

65+
// Use a single iteration in tests to speed up execution, otherwise use 5 seconds
66+
let benchtime = if cfg!(test) { "1x" } else { "5s" };
67+
6568
let cmd = std::process::Command::new(bench_path)
6669
.arg(format!("-test.bench={bench}"))
67-
.arg("-test.benchtime=100x")
70+
.arg(format!("-test.benchtime={benchtime}"))
6871
.output()
6972
.context("Failed to execute benchmark command")?;
7073

‎go-runner/src/results/walltime_results.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ struct BenchmarkConfig {
4444
#[derive(Debug, Serialize, Deserialize)]
4545
pub struct WalltimeBenchmark {
4646
#[serde(flatten)]
47-
metadata: BenchmarkMetadata,
47+
pub metadata: BenchmarkMetadata,
4848

4949
config: BenchmarkConfig,
5050
stats: BenchmarkStats,
@@ -151,7 +151,7 @@ pub struct Creator {
151151
pub struct WalltimeResults {
152152
creator: Creator,
153153
instrument: Instrument,
154-
benchmarks: Vec<WalltimeBenchmark>,
154+
pub benchmarks: Vec<WalltimeBenchmark>,
155155
}
156156

157157
impl WalltimeResults {

‎go-runner/src/snapshots/go_runner__integration_tests__build_and_run.snap renamed to ‎go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@fzf_0.snap

File renamed without changes.

‎go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@golang-benchmarks_0.snap

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
source: src/integration_tests.rs
3+
expression: content
4+
---
5+
{
6+
"creator": {
7+
"name": "codspeed-go",
8+
"version": "0.1.0",
9+
"pid": "[pid]"
10+
},
11+
"instrument": {
12+
"type": "walltime"
13+
},
14+
"benchmarks": [
15+
{
16+
"name": "BenchmarkBase64decode",
17+
"uri": "base64/base64_test.go::BenchmarkBase64decode",
18+
"config": {
19+
"warmup_time_ns": null,
20+
"min_round_time_ns": null,
21+
"max_time_ns": null,
22+
"max_rounds": null
23+
},
24+
"stats": "[stats]"
25+
},
26+
{
27+
"name": "BenchmarkBase64regex",
28+
"uri": "base64/base64_test.go::BenchmarkBase64regex",
29+
"config": {
30+
"warmup_time_ns": null,
31+
"min_round_time_ns": null,
32+
"max_time_ns": null,
33+
"max_rounds": null
34+
},
35+
"stats": "[stats]"
36+
}
37+
]
38+
}

‎go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@golang-benchmarks_1.snap

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
---
2+
source: src/integration_tests.rs
3+
expression: content
4+
---
5+
{
6+
"creator": {
7+
"name": "codspeed-go",
8+
"version": "0.1.0",
9+
"pid": "[pid]"
10+
},
11+
"instrument": {
12+
"type": "walltime"
13+
},
14+
"benchmarks": [
15+
{
16+
"name": "BenchmarkCompileMatch",
17+
"uri": "contains/contains_test.go::BenchmarkCompileMatch",
18+
"config": {
19+
"warmup_time_ns": null,
20+
"min_round_time_ns": null,
21+
"max_time_ns": null,
22+
"max_rounds": null
23+
},
24+
"stats": "[stats]"
25+
},
26+
{
27+
"name": "BenchmarkCompileMatchNot",
28+
"uri": "contains/contains_test.go::BenchmarkCompileMatchNot",
29+
"config": {
30+
"warmup_time_ns": null,
31+
"min_round_time_ns": null,
32+
"max_time_ns": null,
33+
"max_rounds": null
34+
},
35+
"stats": "[stats]"
36+
},
37+
{
38+
"name": "BenchmarkContains",
39+
"uri": "contains/contains_test.go::BenchmarkContains",
40+
"config": {
41+
"warmup_time_ns": null,
42+
"min_round_time_ns": null,
43+
"max_time_ns": null,
44+
"max_rounds": null
45+
},
46+
"stats": "[stats]"
47+
},
48+
{
49+
"name": "BenchmarkContainsBytes",
50+
"uri": "contains/contains_test.go::BenchmarkContainsBytes",
51+
"config": {
52+
"warmup_time_ns": null,
53+
"min_round_time_ns": null,
54+
"max_time_ns": null,
55+
"max_rounds": null
56+
},
57+
"stats": "[stats]"
58+
},
59+
{
60+
"name": "BenchmarkContainsBytesNot",
61+
"uri": "contains/contains_test.go::BenchmarkContainsBytesNot",
62+
"config": {
63+
"warmup_time_ns": null,
64+
"min_round_time_ns": null,
65+
"max_time_ns": null,
66+
"max_rounds": null
67+
},
68+
"stats": "[stats]"
69+
},
70+
{
71+
"name": "BenchmarkContainsMethods::Bytes.Contains",
72+
"uri": "unknown::BenchmarkContainsMethods::Bytes.Contains",
73+
"config": {
74+
"warmup_time_ns": null,
75+
"min_round_time_ns": null,
76+
"max_time_ns": null,
77+
"max_rounds": null
78+
},
79+
"stats": "[stats]"
80+
},
81+
{
82+
"name": "BenchmarkContainsMethods::RegexMatch",
83+
"uri": "unknown::BenchmarkContainsMethods::RegexMatch",
84+
"config": {
85+
"warmup_time_ns": null,
86+
"min_round_time_ns": null,
87+
"max_time_ns": null,
88+
"max_rounds": null
89+
},
90+
"stats": "[stats]"
91+
},
92+
{
93+
"name": "BenchmarkContainsMethods::RegexMatchString",
94+
"uri": "unknown::BenchmarkContainsMethods::RegexMatchString",
95+
"config": {
96+
"warmup_time_ns": null,
97+
"min_round_time_ns": null,
98+
"max_time_ns": null,
99+
"max_rounds": null
100+
},
101+
"stats": "[stats]"
102+
},
103+
{
104+
"name": "BenchmarkContainsMethods::Strings.Contains",
105+
"uri": "unknown::BenchmarkContainsMethods::Strings.Contains",
106+
"config": {
107+
"warmup_time_ns": null,
108+
"min_round_time_ns": null,
109+
"max_time_ns": null,
110+
"max_rounds": null
111+
},
112+
"stats": "[stats]"
113+
},
114+
{
115+
"name": "BenchmarkContainsNot",
116+
"uri": "contains/contains_test.go::BenchmarkContainsNot",
117+
"config": {
118+
"warmup_time_ns": null,
119+
"min_round_time_ns": null,
120+
"max_time_ns": null,
121+
"max_rounds": null
122+
},
123+
"stats": "[stats]"
124+
},
125+
{
126+
"name": "BenchmarkMatch",
127+
"uri": "contains/contains_test.go::BenchmarkMatch",
128+
"config": {
129+
"warmup_time_ns": null,
130+
"min_round_time_ns": null,
131+
"max_time_ns": null,
132+
"max_rounds": null
133+
},
134+
"stats": "[stats]"
135+
},
136+
{
137+
"name": "BenchmarkMatchNot",
138+
"uri": "contains/contains_test.go::BenchmarkMatchNot",
139+
"config": {
140+
"warmup_time_ns": null,
141+
"min_round_time_ns": null,
142+
"max_time_ns": null,
143+
"max_rounds": null
144+
},
145+
"stats": "[stats]"
146+
}
147+
]
148+
}

‎go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@golang-benchmarks_10.snap

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
---
2+
source: src/integration_tests.rs
3+
expression: content
4+
---
5+
{
6+
"creator": {
7+
"name": "codspeed-go",
8+
"version": "0.1.0",
9+
"pid": "[pid]"
10+
},
11+
"instrument": {
12+
"type": "walltime"
13+
},
14+
"benchmarks": [
15+
{
16+
"name": "BenchmarkMapIntIndex",
17+
"uri": "index/index_test.go::BenchmarkMapIntIndex",
18+
"config": {
19+
"warmup_time_ns": null,
20+
"min_round_time_ns": null,
21+
"max_time_ns": null,
22+
"max_rounds": null
23+
},
24+
"stats": "[stats]"
25+
},
26+
{
27+
"name": "BenchmarkMapIntKeys",
28+
"uri": "index/index_test.go::BenchmarkMapIntKeys",
29+
"config": {
30+
"warmup_time_ns": null,
31+
"min_round_time_ns": null,
32+
"max_time_ns": null,
33+
"max_rounds": null
34+
},
35+
"stats": "[stats]"
36+
},
37+
{
38+
"name": "BenchmarkMapStringIndex",
39+
"uri": "index/index_test.go::BenchmarkMapStringIndex",
40+
"config": {
41+
"warmup_time_ns": null,
42+
"min_round_time_ns": null,
43+
"max_time_ns": null,
44+
"max_rounds": null
45+
},
46+
"stats": "[stats]"
47+
},
48+
{
49+
"name": "BenchmarkMapStringKeys",
50+
"uri": "index/index_test.go::BenchmarkMapStringKeys",
51+
"config": {
52+
"warmup_time_ns": null,
53+
"min_round_time_ns": null,
54+
"max_time_ns": null,
55+
"max_rounds": null
56+
},
57+
"stats": "[stats]"
58+
}
59+
]
60+
}

0 commit comments

Comments
 (0)