Skip to content

Commit dca807f

Browse files
author
Artem Eroshenko
authored
fix testng tests execution order (via #578)
1 parent 69af93b commit dca807f

File tree

6 files changed

+138
-28
lines changed

6 files changed

+138
-28
lines changed

‎.github/workflows/build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,4 @@ jobs:
3131

3232
- name: Run tests
3333
if: always()
34-
run: ./gradlew test
34+
run: ./gradlew --no-build-cache cleanTest test

‎allure-java-commons-test/src/main/java/io/qameta/allure/test/AllureFeatures.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,4 +230,12 @@
230230
@Feature("Filtration")
231231
@interface Filtration {
232232
}
233+
234+
@Documented
235+
@Inherited
236+
@Retention(RetentionPolicy.RUNTIME)
237+
@Target({ElementType.METHOD, ElementType.TYPE})
238+
@Feature("Ordering")
239+
@interface Ordering {
240+
}
233241
}

‎allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,8 @@ public class AllureTestNg implements
144144
private final AllureLifecycle lifecycle;
145145
private final AllureTestNgTestFilter testFilter;
146146

147-
public AllureTestNg(final AllureLifecycle lifecycle, final AllureTestNgTestFilter testFilter) {
147+
public AllureTestNg(final AllureLifecycle lifecycle,
148+
final AllureTestNgTestFilter testFilter) {
148149
this.lifecycle = lifecycle;
149150
this.testFilter = testFilter;
150151
}
@@ -154,15 +155,16 @@ public AllureTestNg(final AllureLifecycle lifecycle) {
154155
}
155156

156157
public AllureTestNg() {
157-
this(Allure.getLifecycle(), new AllureTestNgTestFilter());
158+
this(Allure.getLifecycle());
158159
}
159160

160161
public AllureLifecycle getLifecycle() {
161162
return lifecycle;
162163
}
163164

164165
@Override
165-
public List<IMethodInstance> intercept(final List<IMethodInstance> methods, final ITestContext context) {
166+
public List<IMethodInstance> intercept(final List<IMethodInstance> methods,
167+
final ITestContext context) {
166168
return testFilter.intercept(methods, context);
167169
}
168170

‎allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNgTestFilter.java‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,18 @@
2020
import io.qameta.allure.testfilter.TestPlan;
2121
import io.qameta.allure.testfilter.TestPlanUnknown;
2222
import io.qameta.allure.testfilter.TestPlanV1_0;
23-
24-
import java.lang.reflect.Method;
25-
import java.util.List;
26-
import java.util.Optional;
27-
import java.util.stream.Collectors;
28-
2923
import org.testng.IMethodInstance;
3024
import org.testng.IMethodInterceptor;
3125
import org.testng.ITestContext;
3226
import org.testng.ITestNGMethod;
3327
import org.testng.internal.ConstructorOrMethod;
3428

29+
import java.lang.reflect.Method;
30+
import java.util.Comparator;
31+
import java.util.List;
32+
import java.util.Optional;
33+
import java.util.stream.Collectors;
34+
3535
public class AllureTestNgTestFilter implements IMethodInterceptor {
3636

3737
private final TestPlan testPlan;
@@ -46,18 +46,23 @@ public AllureTestNgTestFilter(final TestPlan testPlan) {
4646

4747
@Override
4848
public List<IMethodInstance> intercept(final List<IMethodInstance> methods, final ITestContext context) {
49-
if (testPlan instanceof TestPlanV1_0) {
50-
return methods.stream()
51-
.filter(instance -> isSelected(instance.getMethod()))
52-
.collect(Collectors.toList());
53-
} else {
54-
return methods;
55-
}
49+
return methods.stream()
50+
.filter(this::isSelected)
51+
.sorted(Comparator.comparing(
52+
IMethodInstance::getMethod,
53+
Comparator.nullsFirst(Comparator.comparingInt(
54+
ITestNGMethod::getPriority
55+
))))
56+
.collect(Collectors.toList());
57+
}
58+
59+
public boolean isSelected(final IMethodInstance instance) {
60+
return isSelected(instance.getMethod());
5661
}
5762

58-
public boolean isSelected(final ITestNGMethod instance) {
63+
public boolean isSelected(final ITestNGMethod method) {
5964
if (testPlan instanceof TestPlanV1_0) {
60-
return isSelected(instance, (TestPlanV1_0) testPlan);
65+
return isSelected(method, (TestPlanV1_0) testPlan);
6166
} else {
6267
return true;
6368
}

‎allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import io.qameta.allure.test.AllureResultsWriterStub;
3939
import io.qameta.allure.testfilter.TestPlan;
4040
import io.qameta.allure.testfilter.TestPlanV1_0;
41+
import io.qameta.allure.testng.samples.PriorityTests;
4142
import io.qameta.allure.testng.samples.TestsWithIdForFilter;
4243
import org.assertj.core.api.Condition;
4344
import org.assertj.core.groups.Tuple;
@@ -50,6 +51,7 @@
5051
import java.net.URL;
5152
import java.util.Arrays;
5253
import java.util.Collection;
54+
import java.util.Comparator;
5355
import java.util.List;
5456
import java.util.Objects;
5557
import java.util.Optional;
@@ -434,11 +436,20 @@ public void skippedSuiteTest() {
434436
List<TestResultContainer> testContainers = results.getTestResultContainers();
435437
assertThat(testResults)
436438
.extracting(TestResult::getName, TestResult::getStatus)
437-
.contains(
439+
.containsExactlyInAnyOrder(
438440
tuple("skippedTest", Status.SKIPPED),
441+
tuple("skipSuite", Status.BROKEN),
439442
tuple("testWithOneStep", Status.SKIPPED)
440443
);
441-
assertThat(testContainers).as("Unexpected quantity of testng containers has been written").hasSize(4);
444+
assertThat(testContainers)
445+
.as("Unexpected quantity of testng containers has been written")
446+
.extracting(TestResultContainer::getName)
447+
.containsExactlyInAnyOrder(
448+
"Test tag 8",
449+
"Test suite 8",
450+
"io.qameta.allure.testng.samples.SkippedSuite",
451+
"io.qameta.allure.testng.samples.TestsWithSteps"
452+
);
442453

443454
assertThat(findTestContainerByName(results, "Test suite 8").getBefores())
444455
.as("Before suite container should have a before method with one step")
@@ -677,7 +688,7 @@ public void ownerTest() {
677688
.filter(label -> "owner".equals(label.getName()))
678689
.map(Label::getValue)
679690
.sorted()
680-
.collect(Collectors.joining(",","[","]"))
691+
.collect(Collectors.joining(",", "[", "]"))
681692
)
682693
.containsExactlyInAnyOrder(
683694
tuple("io.qameta.allure.testng.samples.OwnerMethodTest.testWithOwner", "[charlie]"),
@@ -1141,8 +1152,6 @@ public Object[][] failedFixtures() {
11411152
@Test(dataProvider = "failedFixtures")
11421153
@AllureFeatures.Fixtures
11431154
public void shouldAddBeforeFixtureToFakeTestResult(final String suite, final String fixture) {
1144-
System.out.println(suite);
1145-
System.out.println(fixture);
11461155
final AllureResults results = runTestNgSuites(suite);
11471156
final Optional<TestResult> result = results.getTestResults().stream()
11481157
.filter(r -> r.getName().contains(fixture))
@@ -1156,6 +1165,19 @@ public void shouldAddBeforeFixtureToFakeTestResult(final String suite, final Str
11561165
.contains(result.get().getUuid());
11571166
}
11581167

1168+
@Test
1169+
@AllureFeatures.Ordering
1170+
public void shouldOrderTests() {
1171+
final AllureResults results = runTestPlan(null, PriorityTests.class);
1172+
final List<String> ordered = results.getTestResults().stream()
1173+
.sorted(Comparator.comparing(this::getOrderParameter))
1174+
.map(TestResult::getName)
1175+
.collect(Collectors.toList());
1176+
System.out.println(Arrays.toString(ordered.toArray()));
1177+
assertThat(ordered)
1178+
.containsExactly("zTest", "yTest", "xTest", "wTest", "vTest", "vTest");
1179+
}
1180+
11591181
@Step("Run testng suites {suites}")
11601182
private AllureResults runTestNgSuites(final Consumer<TestNG> configurer,
11611183
final String... suites) {
@@ -1338,7 +1360,7 @@ private static void checkTestJavadocDescriptions(List<TestResult> results, Strin
13381360

13391361
@Test
13401362
@AllureFeatures.Filtration
1341-
public void simpleFiltration() {
1363+
public void simpleFiltration() {
13421364
TestPlanV1_0 plan = new TestPlanV1_0().setTests(Arrays.asList(test1, test2, test3));
13431365
List<TestResult> testResults = runTestPlan(plan, TestsWithIdForFilter.class).getTestResults();
13441366

@@ -1354,7 +1376,7 @@ public void simpleFiltration() {
13541376

13551377
@Test
13561378
@AllureFeatures.Filtration
1357-
public void onlyId() {
1379+
public void onlyId() {
13581380
TestPlanV1_0 plan = new TestPlanV1_0().setTests(Arrays.asList(onlyId2, onlyId4));
13591381
List<TestResult> testResults = runTestPlan(plan, TestsWithIdForFilter.class).getTestResults();
13601382

@@ -1397,7 +1419,7 @@ public void skippedTest() {
13971419

13981420
@Test
13991421
@AllureFeatures.Filtration
1400-
public void correctIdIncorrectSelector() {
1422+
public void correctIdIncorrectSelector() {
14011423
TestPlanV1_0 plan = new TestPlanV1_0().setTests(
14021424
Arrays.asList(test1, test2, correctIdIncorrectSelector, correctIdIncorrectSelectorFailed)
14031425
);
@@ -1413,7 +1435,7 @@ public void correctIdIncorrectSelector() {
14131435
);
14141436
}
14151437

1416-
public AllureResultsWriterStub runTestPlan(TestPlan plan, final Class<?> ... testClasses) {
1438+
public AllureResultsWriterStub runTestPlan(TestPlan plan, final Class<?>... testClasses) {
14171439
final AllureResultsWriterStub results = new AllureResultsWriterStub();
14181440
final AllureLifecycle lifecycle = new AllureLifecycle(results);
14191441
final AllureTestNg adapter = new AllureTestNg(lifecycle, new AllureTestNgTestFilter(plan));
@@ -1437,4 +1459,13 @@ public AllureResultsWriterStub runTestPlan(TestPlan plan, final Class<?> ... tes
14371459
}
14381460
}
14391461

1462+
private Integer getOrderParameter(final TestResult result) {
1463+
return result.getParameters().stream()
1464+
.filter(p -> p.getName().equals("order"))
1465+
.map(Parameter::getValue)
1466+
.map(Integer::parseInt)
1467+
.findAny()
1468+
.orElse(0);
1469+
}
1470+
14401471
}

‎allure-testng/src/test/java/io/qameta/allure/testng/samples/PriorityTests.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Copyright 2021 Qameta Software OÜ
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.qameta.allure.testng.samples;
17+
18+
import org.testng.annotations.DataProvider;
19+
import org.testng.annotations.Test;
20+
21+
import java.util.concurrent.atomic.AtomicInteger;
22+
23+
import static io.qameta.allure.Allure.parameter;
24+
25+
public class PriorityTests {
26+
27+
private final static String ORDER_PARAMETER = "order";
28+
29+
private final AtomicInteger cnt = new AtomicInteger();
30+
31+
@DataProvider(name = "someProvider")
32+
public static Object[][] someProvider() {
33+
return new Object[][]{
34+
new Object[]{1},
35+
new Object[]{2}
36+
};
37+
}
38+
39+
@Test(dataProvider = "someProvider", priority = 4)
40+
public void vTest(int parameter) {
41+
parameter(ORDER_PARAMETER, cnt.incrementAndGet());
42+
}
43+
44+
@Test(priority = 3)
45+
public void wTest() {
46+
parameter(ORDER_PARAMETER, cnt.incrementAndGet());
47+
}
48+
49+
@Test(priority = 2)
50+
public void xTest() {
51+
parameter(ORDER_PARAMETER, cnt.incrementAndGet());
52+
}
53+
54+
@Test(priority = 1)
55+
public void yTest() {
56+
parameter(ORDER_PARAMETER, cnt.incrementAndGet());
57+
}
58+
59+
@Test
60+
public void zTest() {
61+
parameter(ORDER_PARAMETER, cnt.incrementAndGet());
62+
}
63+
64+
}

0 commit comments

Comments
 (0)