Skip to content

Commit 29e9697

Browse files
svix-jplattesunshowers
svix-jplatte
and
sunshowers
committed
[cargo-nextest] add support for --skip and --exact test binary arguments
Add support for `--skip` and `--exact` as emulated test binary arguments. Logically: * `--skip` is checked first -- if something matches a skip pattern, it is skipped. * `--exact` is checked similar to regular patterns, except exactly. * Filtersets are intersected with these as usual. Co-authored-by: Rain <[email protected]>
1 parent 1875e84 commit 29e9697

File tree

8 files changed

+743
-118
lines changed

8 files changed

+743
-118
lines changed

‎cargo-nextest/src/dispatch.rs‎

Lines changed: 77 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ use nextest_runner::{
4242
show_config::{ShowNextestVersion, ShowTestGroupSettings, ShowTestGroups, ShowTestGroupsMode},
4343
signal::SignalHandlerKind,
4444
target_runner::{PlatformRunner, TargetRunner},
45-
test_filter::{FilterBound, RunIgnored, TestFilterBuilder},
45+
test_filter::{FilterBound, RunIgnored, TestFilterBuilder, TestFilterPatterns},
4646
write_str::WriteStr,
4747
RustcCli,
4848
};
@@ -590,11 +590,17 @@ struct TestBuildFilter {
590590
#[arg(long)]
591591
ignore_default_filter: bool,
592592

593-
/// Test name filters
593+
/// Test name filters.
594594
#[arg(help_heading = None, name = "FILTERS")]
595595
pre_double_dash_filters: Vec<String>,
596596

597-
/// Test name filters and emulated test binary arguments (partially supported)
597+
/// Test name filters and emulated test binary arguments.
598+
///
599+
/// Arguments supported:{n}
600+
/// - --ignored: Only run ignored tests{n}
601+
/// - --include-ignored: Run both ignored and non-ignored tests{n}
602+
/// - --skip PATTERN: Skip tests that match the pattern{n}
603+
/// - --exact PATTERN: Only run tests that exactly match the pattern
598604
#[arg(help_heading = None, value_name = "FILTERS_AND_ARGS", last = true)]
599605
filters: Vec<String>,
600606
}
@@ -648,53 +654,59 @@ impl TestBuildFilter {
648654
fn make_test_filter_builder(&self, filter_exprs: Vec<Filterset>) -> Result<TestFilterBuilder> {
649655
// Merge the test binary args into the patterns.
650656
let mut run_ignored = self.run_ignored.map(Into::into);
651-
let mut patterns = self.pre_double_dash_filters.clone();
657+
let mut patterns = TestFilterPatterns::new(self.pre_double_dash_filters.clone());
652658
self.merge_test_binary_args(&mut run_ignored, &mut patterns)?;
653659

654660
Ok(TestFilterBuilder::new(
655661
run_ignored.unwrap_or_default(),
656662
self.partition.clone(),
657-
&patterns,
663+
patterns,
658664
filter_exprs,
659665
)?)
660666
}
661667

662668
fn merge_test_binary_args(
663669
&self,
664670
run_ignored: &mut Option<RunIgnored>,
665-
patterns: &mut Vec<String>,
671+
patterns: &mut TestFilterPatterns,
666672
) -> Result<()> {
667673
let mut ignore_filters = Vec::new();
668674
let mut read_trailing_filters = false;
669675

670-
let mut skip_exact = Vec::new();
671676
let mut unsupported_args = Vec::new();
672677

673-
patterns.extend(
674-
self.filters
675-
.iter()
676-
.filter(|&s| {
677-
if read_trailing_filters || !s.starts_with('-') {
678-
true
679-
} else if s == "--include-ignored" {
680-
ignore_filters.push((s.clone(), RunIgnored::All));
681-
false
682-
} else if s == "--ignored" {
683-
ignore_filters.push((s.clone(), RunIgnored::Only));
684-
false
685-
} else if s == "--" {
686-
read_trailing_filters = true;
687-
false
688-
} else if s == "--skip" || s == "--exact" {
689-
skip_exact.push(s.clone());
690-
false
691-
} else {
692-
unsupported_args.push(s.clone());
693-
true
694-
}
695-
})
696-
.cloned(),
697-
);
678+
let mut it = self.filters.iter();
679+
while let Some(arg) = it.next() {
680+
if read_trailing_filters || !arg.starts_with('-') {
681+
patterns.add_substring_pattern(arg.clone());
682+
} else if arg == "--include-ignored" {
683+
ignore_filters.push((arg.clone(), RunIgnored::All));
684+
} else if arg == "--ignored" {
685+
ignore_filters.push((arg.clone(), RunIgnored::Only));
686+
} else if arg == "--" {
687+
read_trailing_filters = true;
688+
} else if arg == "--skip" {
689+
let skip_arg = it.next().ok_or_else(|| {
690+
ExpectedError::test_binary_args_parse_error(
691+
"missing required argument",
692+
vec![arg.clone()],
693+
)
694+
})?;
695+
696+
patterns.add_skip_pattern(skip_arg.clone());
697+
} else if arg == "--exact" {
698+
let exact_arg = it.next().ok_or_else(|| {
699+
ExpectedError::test_binary_args_parse_error(
700+
"missing required argument",
701+
vec![arg.clone()],
702+
)
703+
})?;
704+
705+
patterns.add_exact_pattern(exact_arg.clone());
706+
} else {
707+
unsupported_args.push(arg.clone());
708+
}
709+
}
698710

699711
for (s, f) in ignore_filters {
700712
if let Some(run_ignored) = run_ignored {
@@ -714,19 +726,13 @@ impl TestBuildFilter {
714726
}
715727
}
716728

717-
if !skip_exact.is_empty() {
718-
return Err(ExpectedError::test_binary_args_parse_error(
719-
"unsupported\n(hint: use a filterset instead: <https://nexte.st/docs/filtersets>)",
720-
skip_exact,
721-
));
722-
}
723-
724729
if !unsupported_args.is_empty() {
725730
return Err(ExpectedError::test_binary_args_parse_error(
726731
"unsupported",
727732
unsupported_args,
728733
));
729734
}
735+
730736
Ok(())
731737
}
732738
}
@@ -2594,6 +2600,22 @@ mod tests {
25942600
),
25952601
("foo -- -- str1 str2 --", "foo str1 str2 -- -- --"),
25962602
];
2603+
let skip_exact = &[
2604+
// ---
2605+
// skip and exact
2606+
// ---
2607+
(
2608+
"foo -- --skip my-pattern --skip your-pattern --exact exact1 pattern2",
2609+
{
2610+
let mut patterns = TestFilterPatterns::default();
2611+
patterns.add_skip_pattern("my-pattern".to_owned());
2612+
patterns.add_skip_pattern("your-pattern".to_owned());
2613+
patterns.add_exact_pattern("exact1".to_owned());
2614+
patterns.add_substring_pattern("pattern2".to_owned());
2615+
patterns
2616+
},
2617+
),
2618+
];
25972619
let invalid = &[
25982620
// ---
25992621
// duplicated
@@ -2606,17 +2628,14 @@ mod tests {
26062628
("foo -- --ignored --include-ignored", "mutually exclusive"),
26072629
("foo --run-ignored all -- --ignored", "mutually exclusive"),
26082630
// ---
2631+
// missing required argument
2632+
// ---
2633+
("foo -- --skip", "missing required argument"),
2634+
("foo -- --exact", "missing required argument"),
2635+
// ---
26092636
// unsupported
26102637
// ---
26112638
("foo -- --bar", "unsupported"),
2612-
(
2613-
"foo -- --exact",
2614-
"unsupported\n(hint: use a filterset instead: <https://nexte.st/docs/filtersets>)",
2615-
),
2616-
(
2617-
"foo -- --skip",
2618-
"unsupported\n(hint: use a filterset instead: <https://nexte.st/docs/filtersets>)",
2619-
),
26202639
];
26212640

26222641
for (a, b) in valid {
@@ -2631,6 +2650,17 @@ mod tests {
26312650
assert_eq!(a_str, b_str);
26322651
}
26332652

2653+
for (args, patterns) in skip_exact {
2654+
let builder =
2655+
get_test_filter_builder(args).unwrap_or_else(|_| panic!("failed to parse {args}"));
2656+
2657+
let builder2 =
2658+
TestFilterBuilder::new(RunIgnored::Default, None, patterns.clone(), Vec::new())
2659+
.unwrap_or_else(|_| panic!("failed to build TestFilterBuilder"));
2660+
2661+
assert_eq!(builder, builder2, "{args} matches expected");
2662+
}
2663+
26342664
for (s, r) in invalid {
26352665
let res = get_test_filter_builder(s);
26362666
if let Err(ExpectedError::TestBinaryArgsParseError { reason, .. }) = &res {

‎fixture-data/src/models.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,4 +105,6 @@ impl TestCaseFixtureStatus {
105105
pub enum TestCaseFixtureProperty {
106106
NeedsSameCwd = 1,
107107
NotInDefaultSet = 2,
108+
MatchesCdylib = 4,
109+
MatchesTestMultiplyTwo = 8,
108110
}

‎fixture-data/src/nextest_tests.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,8 @@ pub static EXPECTED_TEST_SUITES: Lazy<BTreeMap<RustBinaryId, TestSuiteFixture>>
139139
"cdylib-link",
140140
BuildPlatform::Target,
141141
vec![
142-
TestCaseFixture::new("test_multiply_two", TestCaseFixtureStatus::Pass),
142+
TestCaseFixture::new("test_multiply_two", TestCaseFixtureStatus::Pass)
143+
.with_property(TestCaseFixtureProperty::MatchesTestMultiplyTwo),
143144
],
144145
),
145146
"dylib-test".into() => TestSuiteFixture::new(
@@ -153,7 +154,9 @@ pub static EXPECTED_TEST_SUITES: Lazy<BTreeMap<RustBinaryId, TestSuiteFixture>>
153154
"cdylib-example",
154155
BuildPlatform::Target,
155156
vec![
156-
TestCaseFixture::new("tests::test_multiply_two_cdylib", TestCaseFixtureStatus::Pass),
157+
TestCaseFixture::new("tests::test_multiply_two_cdylib", TestCaseFixtureStatus::Pass)
158+
.with_property(TestCaseFixtureProperty::MatchesCdylib)
159+
.with_property(TestCaseFixtureProperty::MatchesTestMultiplyTwo),
157160
],
158161
)
159162
.with_property(TestSuiteFixtureProperty::NotInDefaultSet),

‎integration-tests/tests/integration/fixtures.rs‎

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,10 @@ impl CheckResult {
314314
pub enum RunProperty {
315315
Relocated = 1,
316316
WithDefaultFilter = 2,
317+
// --skip cdylib
318+
WithSkipCdylibFilter = 4,
319+
// --exact test_multiply_two tests::test_multiply_two_cdylib
320+
WithMultiplyTwoExactFilter = 8,
317321
}
318322

319323
fn debug_run_properties(properties: u64) -> String {
@@ -324,6 +328,12 @@ fn debug_run_properties(properties: u64) -> String {
324328
if properties & RunProperty::WithDefaultFilter as u64 != 0 {
325329
ret.push_str("with-default-filter ");
326330
}
331+
if properties & RunProperty::WithSkipCdylibFilter as u64 != 0 {
332+
ret.push_str("with-skip-cdylib-filter ");
333+
}
334+
if properties & RunProperty::WithMultiplyTwoExactFilter as u64 != 0 {
335+
ret.push_str("with-exact-filter ");
336+
}
327337
ret
328338
}
329339

@@ -372,6 +382,28 @@ pub fn check_run_output(stderr: &[u8], properties: u64) {
372382
skip_count += 1;
373383
continue;
374384
}
385+
if test.has_property(TestCaseFixtureProperty::MatchesCdylib)
386+
&& properties & RunProperty::WithSkipCdylibFilter as u64 != 0
387+
{
388+
eprintln!("*** skipping {name}");
389+
assert!(
390+
!output.contains(&name),
391+
"test '{name}' should not be run with --skip cdylib"
392+
);
393+
skip_count += 1;
394+
continue;
395+
}
396+
if !test.has_property(TestCaseFixtureProperty::MatchesTestMultiplyTwo)
397+
&& properties & RunProperty::WithMultiplyTwoExactFilter as u64 != 0
398+
{
399+
eprintln!("*** skipping {name}");
400+
assert!(
401+
!output.contains(&name),
402+
"test '{name}' should not be run with --exact test_multiply_two test_multiply_two_cdylib"
403+
);
404+
skip_count += 1;
405+
continue;
406+
}
375407

376408
let result = match test.status {
377409
// This is not a complete accounting -- for example, the needs-same-cwd check should
@@ -423,9 +455,30 @@ pub fn check_run_output(stderr: &[u8], properties: u64) {
423455
}
424456
}
425457

426-
let summary_regex_str = format!(
427-
r"Summary \[.*\] *{run_count} tests run: {pass_count} passed \({leak_count} leaky\), {fail_count} failed, {skip_count} skipped"
428-
);
458+
let tests_str = if run_count == 1 { "test" } else { "tests" };
459+
460+
let summary_regex_str = match (leak_count, fail_count) {
461+
(0, 0) => {
462+
format!(
463+
r"Summary \[.*\] *{run_count} {tests_str} run: {pass_count} passed, {skip_count} skipped"
464+
)
465+
}
466+
(0, _) => {
467+
format!(
468+
r"Summary \[.*\] *{run_count} {tests_str} run: {pass_count} passed, {fail_count} failed, {skip_count} skipped"
469+
)
470+
}
471+
(_, 0) => {
472+
format!(
473+
r"Summary \[.*\] *{run_count} {tests_str} run: {pass_count} passed \({leak_count} leaky\), {skip_count} skipped"
474+
)
475+
}
476+
(_, _) => {
477+
format!(
478+
r"Summary \[.*\] *{run_count} {tests_str} run: {pass_count} passed \({leak_count} leaky\), {fail_count} failed, {skip_count} skipped"
479+
)
480+
}
481+
};
429482
let summary_reg = Regex::new(&summary_regex_str).unwrap();
430483
assert!(
431484
summary_reg.is_match(&output),

0 commit comments

Comments
 (0)