Skip to content

Commit 55cf82d

Browse files
not-matthiasnot-matthias
committed
feat: add integration tests
1 parent 5de8f5f commit 55cf82d

File tree

33 files changed

+3184
-2
lines changed

33 files changed

+3184
-2
lines changed

‎go-runner/src/builder/discovery.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use std::{
99

1010
use crate::builder::verifier;
1111
use crate::prelude::*;
12+
use itertools::Itertools;
1213
use serde::{Deserialize, Serialize};
1314

1415
/// Represents a Go package, deserialized from `go list -json` output.
@@ -78,7 +79,7 @@ impl GoPackage {
7879
};
7980

8081
let mut benchmarks = Vec::new();
81-
'file_loop: for file in test_go_files {
82+
'file_loop: for file in test_go_files.iter().sorted() {
8283
assert!(file.ends_with("_test.go"));
8384

8485
let file_path = self.dir.join(file);
@@ -166,7 +167,7 @@ impl GoPackage {
166167
continue;
167168
}
168169

169-
for func in valid_benchmarks {
170+
for func in valid_benchmarks.into_iter().sorted() {
170171
benchmarks.push(GoBenchmark::new(
171172
package_import_path.clone(),
172173
func,

‎go-runner/src/integration_tests.rs

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
use itertools::Itertools;
2+
use rstest::rstest;
3+
use std::path::{Path, PathBuf};
4+
use std::sync::Mutex;
5+
use tempfile::TempDir;
6+
7+
use crate::results::walltime_results::WalltimeResults;
8+
9+
fn setup_test_project(project_name: &str) -> anyhow::Result<TempDir> {
10+
let project_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11+
.join("testdata/projects")
12+
.join(project_name);
13+
println!("Project path: {project_path:?}");
14+
15+
let temp_dir = TempDir::new()?;
16+
crate::utils::copy_dir_recursively(&project_path, &temp_dir)?;
17+
18+
Ok(temp_dir)
19+
}
20+
21+
fn assert_results_snapshots(profile_dir: &Path, project_name: &str) {
22+
let glob_pattern = profile_dir.join("results");
23+
if !glob_pattern.exists() {
24+
eprintln!("No results found for project: {project_name}");
25+
return;
26+
}
27+
28+
let files = std::fs::read_dir(&glob_pattern)
29+
.expect("Failed to read results directory")
30+
.filter_map(Result::ok)
31+
.map(|entry| entry.path())
32+
.map(|path| {
33+
let file = std::fs::File::open(&path).unwrap();
34+
serde_json::from_reader::<_, WalltimeResults>(file).unwrap()
35+
})
36+
// Ensure we have the correct order for multiple test executables
37+
.sorted_by_cached_key(|r| {
38+
r.benchmarks
39+
.iter()
40+
.map(|b| b.metadata.name.clone())
41+
.sorted()
42+
.join(";")
43+
})
44+
.collect::<Vec<_>>();
45+
46+
for (i, mut content) in files.into_iter().enumerate() {
47+
content
48+
.benchmarks
49+
.sort_by_cached_key(|b| b.metadata.name.clone());
50+
51+
let _guard = {
52+
let mut settings = insta::Settings::clone_current();
53+
settings.set_snapshot_suffix(format!("{project_name}_{i}"));
54+
settings.bind_to_scope()
55+
};
56+
57+
insta::assert_json_snapshot!(content, {
58+
".creator.pid" => "[pid]",
59+
".benchmarks[].stats" => "[stats]",
60+
});
61+
}
62+
}
63+
64+
#[rstest]
65+
// // #[case::caddy("caddy")]
66+
#[case::fzf("fzf")]
67+
#[case::opentelemetry_go("opentelemetry-go")]
68+
#[case::golang_benchmarks("golang-benchmarks")]
69+
#[case::zerolog("zerolog")]
70+
#[case::zap("zap")]
71+
#[case::hugo("hugo")]
72+
// Currently not producing results:
73+
#[case::fuego("fuego")]
74+
#[case::cli_runtime("cli-runtime")]
75+
fn test_build_and_run(#[case] project_name: &str) {
76+
let temp_dir = setup_test_project(project_name).unwrap();
77+
78+
// Mutex to prevent concurrent tests from interfering with CODSPEED_PROFILE_FOLDER env var
79+
static ENV_MUTEX: Mutex<()> = Mutex::new(());
80+
let _env_guard = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
81+
82+
let profile_dir = temp_dir.path().join("profile");
83+
unsafe { std::env::set_var("CODSPEED_PROFILE_FOLDER", &profile_dir) };
84+
if let Err(error) = crate::run_benchmarks(temp_dir.path(), ".") {
85+
panic!("Benchmarks couldn't run: {error}");
86+
}
87+
88+
assert_results_snapshots(&profile_dir, project_name);
89+
}

‎go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@fzf_0.snap

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
---
2+
source: src/integration_tests.rs
3+
expression: content
4+
---
5+
{
6+
"creator": {
7+
"name": "codspeed-go",
8+
"version": "0.1.0",
9+
"pid": "[pid]"
10+
},
11+
"instrument": {
12+
"type": "walltime"
13+
},
14+
"benchmarks": [
15+
{
16+
"name": "BenchmarkExtractColor",
17+
"uri": "src/ansi_test.go::BenchmarkExtractColor",
18+
"config": {
19+
"warmup_time_ns": null,
20+
"min_round_time_ns": null,
21+
"max_time_ns": null,
22+
"max_rounds": null
23+
},
24+
"stats": "[stats]"
25+
},
26+
{
27+
"name": "BenchmarkNextAnsiEscapeSequence",
28+
"uri": "src/ansi_test.go::BenchmarkNextAnsiEscapeSequence",
29+
"config": {
30+
"warmup_time_ns": null,
31+
"min_round_time_ns": null,
32+
"max_time_ns": null,
33+
"max_rounds": null
34+
},
35+
"stats": "[stats]"
36+
},
37+
{
38+
"name": "BenchmarkNextAnsiEscapeSequence_Regex",
39+
"uri": "src/ansi_test.go::BenchmarkNextAnsiEscapeSequence_Regex",
40+
"config": {
41+
"warmup_time_ns": null,
42+
"min_round_time_ns": null,
43+
"max_time_ns": null,
44+
"max_rounds": null
45+
},
46+
"stats": "[stats]"
47+
}
48+
]
49+
}

‎go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@golang-benchmarks_0.snap

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
source: src/integration_tests.rs
3+
expression: content
4+
---
5+
{
6+
"creator": {
7+
"name": "codspeed-go",
8+
"version": "0.1.0",
9+
"pid": "[pid]"
10+
},
11+
"instrument": {
12+
"type": "walltime"
13+
},
14+
"benchmarks": [
15+
{
16+
"name": "BenchmarkBase64decode",
17+
"uri": "base64/base64_test.go::BenchmarkBase64decode",
18+
"config": {
19+
"warmup_time_ns": null,
20+
"min_round_time_ns": null,
21+
"max_time_ns": null,
22+
"max_rounds": null
23+
},
24+
"stats": "[stats]"
25+
},
26+
{
27+
"name": "BenchmarkBase64regex",
28+
"uri": "base64/base64_test.go::BenchmarkBase64regex",
29+
"config": {
30+
"warmup_time_ns": null,
31+
"min_round_time_ns": null,
32+
"max_time_ns": null,
33+
"max_rounds": null
34+
},
35+
"stats": "[stats]"
36+
}
37+
]
38+
}

‎go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@golang-benchmarks_1.snap

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
---
2+
source: src/integration_tests.rs
3+
expression: content
4+
---
5+
{
6+
"creator": {
7+
"name": "codspeed-go",
8+
"version": "0.1.0",
9+
"pid": "[pid]"
10+
},
11+
"instrument": {
12+
"type": "walltime"
13+
},
14+
"benchmarks": [
15+
{
16+
"name": "BenchmarkCompileMatch",
17+
"uri": "contains/contains_test.go::BenchmarkCompileMatch",
18+
"config": {
19+
"warmup_time_ns": null,
20+
"min_round_time_ns": null,
21+
"max_time_ns": null,
22+
"max_rounds": null
23+
},
24+
"stats": "[stats]"
25+
},
26+
{
27+
"name": "BenchmarkCompileMatchNot",
28+
"uri": "contains/contains_test.go::BenchmarkCompileMatchNot",
29+
"config": {
30+
"warmup_time_ns": null,
31+
"min_round_time_ns": null,
32+
"max_time_ns": null,
33+
"max_rounds": null
34+
},
35+
"stats": "[stats]"
36+
},
37+
{
38+
"name": "BenchmarkContains",
39+
"uri": "contains/contains_test.go::BenchmarkContains",
40+
"config": {
41+
"warmup_time_ns": null,
42+
"min_round_time_ns": null,
43+
"max_time_ns": null,
44+
"max_rounds": null
45+
},
46+
"stats": "[stats]"
47+
},
48+
{
49+
"name": "BenchmarkContainsBytes",
50+
"uri": "contains/contains_test.go::BenchmarkContainsBytes",
51+
"config": {
52+
"warmup_time_ns": null,
53+
"min_round_time_ns": null,
54+
"max_time_ns": null,
55+
"max_rounds": null
56+
},
57+
"stats": "[stats]"
58+
},
59+
{
60+
"name": "BenchmarkContainsBytesNot",
61+
"uri": "contains/contains_test.go::BenchmarkContainsBytesNot",
62+
"config": {
63+
"warmup_time_ns": null,
64+
"min_round_time_ns": null,
65+
"max_time_ns": null,
66+
"max_rounds": null
67+
},
68+
"stats": "[stats]"
69+
},
70+
{
71+
"name": "BenchmarkContainsMethods::Bytes.Contains",
72+
"uri": "unknown::BenchmarkContainsMethods::Bytes.Contains",
73+
"config": {
74+
"warmup_time_ns": null,
75+
"min_round_time_ns": null,
76+
"max_time_ns": null,
77+
"max_rounds": null
78+
},
79+
"stats": "[stats]"
80+
},
81+
{
82+
"name": "BenchmarkContainsMethods::RegexMatch",
83+
"uri": "unknown::BenchmarkContainsMethods::RegexMatch",
84+
"config": {
85+
"warmup_time_ns": null,
86+
"min_round_time_ns": null,
87+
"max_time_ns": null,
88+
"max_rounds": null
89+
},
90+
"stats": "[stats]"
91+
},
92+
{
93+
"name": "BenchmarkContainsMethods::RegexMatchString",
94+
"uri": "unknown::BenchmarkContainsMethods::RegexMatchString",
95+
"config": {
96+
"warmup_time_ns": null,
97+
"min_round_time_ns": null,
98+
"max_time_ns": null,
99+
"max_rounds": null
100+
},
101+
"stats": "[stats]"
102+
},
103+
{
104+
"name": "BenchmarkContainsMethods::Strings.Contains",
105+
"uri": "unknown::BenchmarkContainsMethods::Strings.Contains",
106+
"config": {
107+
"warmup_time_ns": null,
108+
"min_round_time_ns": null,
109+
"max_time_ns": null,
110+
"max_rounds": null
111+
},
112+
"stats": "[stats]"
113+
},
114+
{
115+
"name": "BenchmarkContainsNot",
116+
"uri": "contains/contains_test.go::BenchmarkContainsNot",
117+
"config": {
118+
"warmup_time_ns": null,
119+
"min_round_time_ns": null,
120+
"max_time_ns": null,
121+
"max_rounds": null
122+
},
123+
"stats": "[stats]"
124+
},
125+
{
126+
"name": "BenchmarkMatch",
127+
"uri": "contains/contains_test.go::BenchmarkMatch",
128+
"config": {
129+
"warmup_time_ns": null,
130+
"min_round_time_ns": null,
131+
"max_time_ns": null,
132+
"max_rounds": null
133+
},
134+
"stats": "[stats]"
135+
},
136+
{
137+
"name": "BenchmarkMatchNot",
138+
"uri": "contains/contains_test.go::BenchmarkMatchNot",
139+
"config": {
140+
"warmup_time_ns": null,
141+
"min_round_time_ns": null,
142+
"max_time_ns": null,
143+
"max_rounds": null
144+
},
145+
"stats": "[stats]"
146+
}
147+
]
148+
}

0 commit comments

Comments
 (0)