From 25b09ccf81d0e5d258ef73b9978bd3553a6139b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Kautler?= Date: Fri, 12 May 2023 19:21:41 +0200 Subject: [PATCH 1/2] Serialize tests during coverage calculation This generically fixes hcoles/pitest#760 and #73 for all platform engines, removing the Jupiter specific work-around from #74 and serializing test execution during coverage calculation using locks. This way the tests can also properly run in parallel later on during mutant hunting. --- .../junit5/JUnit5TestPluginFactory.java | 5 +- .../pitest/junit5/JUnit5TestUnitFinder.java | 165 ++++++++++++++++-- 2 files changed, 151 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/pitest/junit5/JUnit5TestPluginFactory.java b/src/main/java/org/pitest/junit5/JUnit5TestPluginFactory.java index 2cb3a19..fa8dc6c 100755 --- a/src/main/java/org/pitest/junit5/JUnit5TestPluginFactory.java +++ b/src/main/java/org/pitest/junit5/JUnit5TestPluginFactory.java @@ -27,11 +27,10 @@ public class JUnit5TestPluginFactory implements TestPluginFactory { @Override - public Configuration createTestFrameworkConfiguration(TestGroupConfig config, - ClassByteArraySource source, + public Configuration createTestFrameworkConfiguration(TestGroupConfig config, + ClassByteArraySource source, Collection excludedRunners, Collection includedTestMethods) { - System.setProperty("junit.jupiter.execution.parallel.enabled", "false"); return new JUnit5Configuration(config, includedTestMethods); } diff --git a/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java b/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java index c1ac807..d496f9e 100755 --- a/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java +++ b/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java @@ -16,17 +16,25 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import static java.util.Collections.emptyList; -import static java.util.Collections.synchronizedList; +import static java.util.Collections.synchronizedSet; import static java.util.Collections.unmodifiableList; import static java.util.stream.Collectors.toList; import org.junit.platform.commons.PreconditionViolationException; import org.junit.platform.engine.Filter; import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.engine.support.descriptor.MethodSource; import org.junit.platform.launcher.Launcher; @@ -36,6 +44,7 @@ import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; import org.pitest.testapi.Description; +import org.pitest.testapi.NullExecutionListener; import org.pitest.testapi.TestGroupConfig; import org.pitest.testapi.TestUnit; import org.pitest.testapi.TestUnitExecutionListener; @@ -47,12 +56,27 @@ */ public class JUnit5TestUnitFinder implements TestUnitFinder { + /** + * The test group config. + */ private final TestGroupConfig testGroupConfig; + /** + * Test methods that should be included. + */ private final Collection includedTestMethods; + /** + * The JUnit platform launcher used to execute tests. + */ private final Launcher launcher; + /** + * Constructs a new JUnit 5 test unit finder. + * + * @param testGroupConfig the test group config + * @param includedTestMethods test methods that should be included + */ public JUnit5TestUnitFinder(TestGroupConfig testGroupConfig, Collection includedTestMethods) { this.testGroupConfig = testGroupConfig; this.includedTestMethods = includedTestMethods; @@ -65,7 +89,7 @@ public List findTestUnits(Class clazz, TestUnitExecutionListener ex return emptyList(); } - List filters = new ArrayList<>(2); + List> filters = new ArrayList<>(2); try { List excludedGroups = testGroupConfig.getExcludedGroups().stream().filter(group -> !group.isEmpty()).collect(Collectors.toList()); if(!excludedGroups.isEmpty()) { @@ -85,7 +109,7 @@ public List findTestUnits(Class clazz, TestUnitExecutionListener ex launcher.execute(LauncherDiscoveryRequestBuilder .request() .selectors(DiscoverySelectors.selectClass(clazz)) - .filters(filters.toArray(new Filter[filters.size()])) + .filters(filters.toArray(new Filter[0])) .build(), listener); return listener.getIdentifiers() @@ -94,17 +118,92 @@ public List findTestUnits(Class clazz, TestUnitExecutionListener ex .collect(toList()); } + /** + * A test execution listener that listens for test identifiers, supporting atomic test units + * and notifying the supplied test unit execution listener so that for example coverage can + * be recorded right away during discovery phase already. + */ private class TestIdentifierListener implements TestExecutionListener { + /** + * The test class as given to the test unit finder for forwarding to the test unit execution listener. + */ private final Class testClass; - private final TestUnitExecutionListener l; - private final List identifiers = synchronizedList(new ArrayList<>()); - public TestIdentifierListener(Class testClass, TestUnitExecutionListener l) { + /** + * The test unit execution listener, that for example is used for coverage recording per test. + */ + private final TestUnitExecutionListener testUnitExecutionListener; + + /** + * The collected test identifiers. + */ + private final Set identifiers = synchronizedSet(new LinkedHashSet<>()); + + /** + * Whether to serialize test execution, because we are during coverage recording which is + * done through static fields and thus does not support parallel test execution. + */ + private final boolean serializeExecution; + + /** + * A map that holds the locks that child tests of locked parent tests should use. + * For example parallel data-driven Spock features start the feature execution which is CONTAINER_AND_TEST, + * then wait for the parallel iteration executions to be finished which are TEST, + * then finish the feature execution. + * Due to that we cannot lock the iteration executions on the same lock as the feature executions, + * as the feature execution is around all the subordinate iteration executions. + * + *

This logic will of course break if there is some test engine that does strange setups like + * having CONTAINER_AND_TEST with child CONTAINER that have child TEST and similar. + * If those engines happen to be used, tests will start to deadlock, as the grand-child test + * would not find the parent serializer and thus use the root serializer on which the grand-parent + * CONTAINER_AND_TEST already locks. + * + *

This setup would probably not make much sense, so should not be taken into account + * unless such an engine actually pops up. If it does and someone tries to use it with PIT, + * the logic should maybe be made more sophisticated like remembering the parent-child relationships + * to be able to find the grand-parent serializer which is not possible stateless, because we are + * only able to get the parent identifier directly, but not further up stateless. + */ + private final Map> parentCoverageSerializers = new ConcurrentHashMap<>(); + + /** + * A map that holds the actual lock used for a specific test to be able to easily and safely unlock + * without the need to recalculate which lock to use. + */ + private final Map coverageSerializers = new ConcurrentHashMap<>(); + + /** + * The root coverage serializer to be used for the top-most recorded tests. + */ + private final ReentrantLock rootCoverageSerializer = new ReentrantLock(); + + /** + * Constructs a new test identifier listener. + * + * @param testClass the test class as given to the test unit finder for forwarding to the result collector + * @param testUnitExecutionListener the test unit execution listener to notify during test execution + */ + public TestIdentifierListener(Class testClass, TestUnitExecutionListener testUnitExecutionListener) { this.testClass = testClass; - this.l = l; + this.testUnitExecutionListener = testUnitExecutionListener; + // PIT gives a coverage recording listener here during coverage recording + // At the later stage during minion hunting a NullExecutionListener is given + // as PIT is only interested in the resulting list of identifiers. + // Serialization of test execution is only necessary during coverage calculation + // currently. To be on the safe side serialize test execution for any listener + // type except listener types where we know tests can run in parallel safely, + // i.e. currently the NullExecutionListener which is the only other one besides + // the coverage recording listener. + serializeExecution = !(testUnitExecutionListener instanceof NullExecutionListener); } - List getIdentifiers() { + /** + * Returns the collected test identifiers. + * + * @return the collected test identifiers + */ + private List getIdentifiers() { return unmodifiableList(new ArrayList<>(identifiers)); } @@ -118,27 +217,61 @@ public void executionStarted(TestIdentifier testIdentifier) { && !includedTestMethods.contains(((MethodSource)testIdentifier.getSource().get()).getMethodName())) { return; } - l.executionStarted(new Description(testIdentifier.getUniqueId(), testClass)); + + if (serializeExecution) { + coverageSerializers.compute(testIdentifier.getUniqueIdObject(), (uniqueId, lock) -> { + if (lock != null) { + throw new AssertionError("No lock should be present"); + } + + // find the serializer to lock the test on + // if there is a parent test locked, use the lock for its children if not, + // use the root serializer + return testIdentifier + .getParentIdObject() + .map(parentCoverageSerializers::get) + .map(lockRef -> lockRef.updateAndGet(parentLock -> + parentLock == null ? new ReentrantLock() : parentLock)) + .orElse(rootCoverageSerializer); + }).lock(); + // record a potential serializer for child tests to lock on + parentCoverageSerializers.put(testIdentifier.getUniqueIdObject(), new AtomicReference<>()); + } + + testUnitExecutionListener.executionStarted(new Description(testIdentifier.getUniqueId(), testClass), true); identifiers.add(testIdentifier); } } - @Override public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { // Classes with failing BeforeAlls never start execution and identify as 'containers' not 'tests' if (testExecutionResult.getStatus() == TestExecutionResult.Status.FAILED) { - if (!identifiers.contains(testIdentifier)) { - identifiers.add(testIdentifier); - } - l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass) + identifiers.add(testIdentifier); + testUnitExecutionListener.executionFinished(new Description(testIdentifier.getUniqueId(), testClass) , false, testExecutionResult.getThrowable().orElse(null)); } else if (testIdentifier.isTest()) { - l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass) + testUnitExecutionListener.executionFinished(new Description(testIdentifier.getUniqueId(), testClass) , true); } - } + if (serializeExecution) { + // forget the potential serializer for child tests + parentCoverageSerializers.remove(testIdentifier.getUniqueIdObject()); + // unlock the serializer for the finished tests to let the next test continue + ReentrantLock lock = coverageSerializers.remove(testIdentifier.getUniqueIdObject()); + if (lock != null) { + lock.unlock(); + } + } + } } + @Override + public String toString() { + return new StringJoiner(", ", JUnit5TestUnitFinder.class.getSimpleName() + "[", "]") + .add("testGroupConfig=" + testGroupConfig) + .add("includedTestMethods=" + includedTestMethods) + .toString(); + } } From 75fe891dcf341993c443cd2222254862a617c5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Kautler?= Date: Sun, 14 May 2023 01:08:56 +0200 Subject: [PATCH 2/2] Make Spock specs atomic if certain criteria are met --- pom.xml | 7 + .../pitest/junit5/JUnit5TestUnitFinder.java | 383 +++++++++++++++++- .../repository/TestSpecWithAfterAll.groovy | 34 ++ .../repository/TestSpecWithAfterClass.groovy | 34 ++ .../repository/TestSpecWithBeforeAll.groovy | 34 ++ .../repository/TestSpecWithBeforeClass.groovy | 34 ++ .../TestSpecWithClassRuleField.groovy | 33 ++ .../TestSpecWithClassRuleMethod.groovy | 34 ++ .../TestSpecWithSetupSpecWithoutShared.groovy | 38 ++ .../repository/TestSpecWithShared.groovy | 33 ++ .../repository/TestSpecWithStepwise.groovy | 31 ++ .../TestSpecWithStepwiseFeature.groovy | 31 ++ .../junit5/JUnit5TestUnitFinderTest.java | 78 +++- 13 files changed, 774 insertions(+), 30 deletions(-) create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterAll.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterClass.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeAll.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeClass.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleField.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleMethod.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithSetupSpecWithoutShared.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithShared.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwise.groovy create mode 100644 src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwiseFeature.groovy diff --git a/pom.xml b/pom.xml index 6e8371a..2ec49cb 100755 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,7 @@ 1.15.2 5.0.0 2.3-groovy-4.0 + 4.13.2 4.0.11 @@ -200,6 +201,12 @@ spock-core test + + junit + junit + ${junit4.version} + test + diff --git a/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java b/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java index d496f9e..ab064b3 100755 --- a/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java +++ b/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java @@ -14,16 +14,23 @@ */ package org.pitest.junit5; +import java.lang.annotation.Annotation; +import java.lang.reflect.AnnotatedElement; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Predicate; import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -36,6 +43,7 @@ import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.engine.support.descriptor.ClassSource; import org.junit.platform.engine.support.descriptor.MethodSource; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.TagFilter; @@ -43,6 +51,8 @@ import org.junit.platform.launcher.TestIdentifier; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; +import org.pitest.functional.FCollection; +import org.pitest.reflection.Reflection; import org.pitest.testapi.Description; import org.pitest.testapi.NullExecutionListener; import org.pitest.testapi.TestGroupConfig; @@ -55,6 +65,53 @@ * @author Tobias Stadler */ public class JUnit5TestUnitFinder implements TestUnitFinder { + /** + * The Spock {@code Specification} class. + */ + private static final Optional> SPECIFICATION = + findClass("spock.lang.Specification"); + + /** + * The Jupiter {@code @BeforeAll} annotation. + */ + private static final Optional> BEFORE_ALL = + findClass("org.junit.jupiter.api.BeforeAll"); + + /** + * The JUnit 4 {@code @BeforeClass} annotation. + */ + private static final Optional> BEFORE_CLASS = + findClass("org.junit.BeforeClass"); + + /** + * The Jupiter {@code @AfterAll} annotation. + */ + private static final Optional> AFTER_ALL = + findClass("org.junit.jupiter.api.AfterAll"); + + /** + * The JUnit 4 {@code @AfterClass} annotation. + */ + private static final Optional> AFTER_CLASS = + findClass("org.junit.AfterClass"); + + /** + * The JUnit 4 {@code @ClassRule} annotation. + */ + private static final Optional> CLASS_RULE = + findClass("org.junit.ClassRule"); + + /** + * The Spock {@code @Shared} annotation. + */ + private static final Optional> SHARED = + findClass("spock.lang.Shared"); + + /** + * The Spock {@code @Stepwise} annotation. + */ + private static final Optional> STEPWISE = + findClass("spock.lang.Stepwise"); /** * The test group config. @@ -118,6 +175,22 @@ public List findTestUnits(Class clazz, TestUnitExecutionListener ex .collect(toList()); } + /** + * Finds a class via reflection and returns it if found, or an empty {@code Optional} otherwise. + * + * @param className the name of the class to find + * @return the class if present + * @param the type of the class + */ + @SuppressWarnings("unchecked") + private static Optional> findClass(String className) { + try { + return Optional.of(((Class) Class.forName(className))); + } catch (final ClassNotFoundException ex) { + return Optional.empty(); + } + } + /** * A test execution listener that listens for test identifiers, supporting atomic test units * and notifying the supplied test unit execution listener so that for example coverage can @@ -209,31 +282,21 @@ private List getIdentifiers() { @Override public void executionStarted(TestIdentifier testIdentifier) { + if (shouldTreatAsOneUnit(testIdentifier)) { + executionOfAtomicPartStarted(testIdentifier); + return; + } + if (testIdentifier.isTest()) { // filter out testMethods if (includedTestMethods != null && !includedTestMethods.isEmpty() - && testIdentifier.getSource().isPresent() - && testIdentifier.getSource().get() instanceof MethodSource - && !includedTestMethods.contains(((MethodSource)testIdentifier.getSource().get()).getMethodName())) { + && hasMethodSource(testIdentifier) + && !includedTestMethods.contains(((MethodSource) testIdentifier.getSource().get()).getMethodName())) { return; } if (serializeExecution) { - coverageSerializers.compute(testIdentifier.getUniqueIdObject(), (uniqueId, lock) -> { - if (lock != null) { - throw new AssertionError("No lock should be present"); - } - - // find the serializer to lock the test on - // if there is a parent test locked, use the lock for its children if not, - // use the root serializer - return testIdentifier - .getParentIdObject() - .map(parentCoverageSerializers::get) - .map(lockRef -> lockRef.updateAndGet(parentLock -> - parentLock == null ? new ReentrantLock() : parentLock)) - .orElse(rootCoverageSerializer); - }).lock(); + lock(testIdentifier); // record a potential serializer for child tests to lock on parentCoverageSerializers.put(testIdentifier.getUniqueIdObject(), new AtomicReference<>()); } @@ -243,9 +306,36 @@ public void executionStarted(TestIdentifier testIdentifier) { } } + /** + * Handle the start of execution of an atomic part. + * + * @param testIdentifier the test identifier of the atomic part + */ + private void executionOfAtomicPartStarted(TestIdentifier testIdentifier) { + if (hasClassSource(testIdentifier)) { + if (serializeExecution) { + lock(testIdentifier); + } + + testUnitExecutionListener.executionStarted(new Description(testIdentifier.getUniqueId(), testClass), true); + identifiers.add(testIdentifier); + } + } + @Override public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { - // Classes with failing BeforeAlls never start execution and identify as 'containers' not 'tests' + if (shouldTreatAsOneUnit(testIdentifier)) { + if (hasClassSource(testIdentifier)) { + testUnitExecutionListener.executionFinished(new Description(testIdentifier.getUniqueId(), testClass), + testExecutionResult.getStatus() != TestExecutionResult.Status.FAILED, + testExecutionResult.getThrowable().orElse(null)); + // unlock the serializer for the finished tests to let the next test continue + unlock(testIdentifier); + } + return; + } + + // Jupiter classes with failing BeforeAlls never start execution and identify as 'containers' not 'tests' if (testExecutionResult.getStatus() == TestExecutionResult.Status.FAILED) { identifiers.add(testIdentifier); testUnitExecutionListener.executionFinished(new Description(testIdentifier.getUniqueId(), testClass) @@ -259,11 +349,262 @@ public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult // forget the potential serializer for child tests parentCoverageSerializers.remove(testIdentifier.getUniqueIdObject()); // unlock the serializer for the finished tests to let the next test continue - ReentrantLock lock = coverageSerializers.remove(testIdentifier.getUniqueIdObject()); + unlock(testIdentifier); + } + } + + /** + * Locks the correct serializer lock for the given test identifier, so that all tests are run sequentially. + * + * @param testIdentifier the test identifier to lock for + */ + public void lock(TestIdentifier testIdentifier) { + coverageSerializers.compute(testIdentifier.getUniqueIdObject(), (uniqueId, lock) -> { if (lock != null) { - lock.unlock(); + throw new AssertionError("No lock should be present"); } + + // find the serializer to lock the test on + // if there is a parent test locked, use the lock for its children if not, + // use the root serializer + return testIdentifier + .getParentIdObject() + .map(parentCoverageSerializers::get) + .map(lockRef -> lockRef.updateAndGet(parentLock -> + parentLock == null ? new ReentrantLock() : parentLock)) + .orElse(rootCoverageSerializer); + }).lock(); + } + + /** + * Unlocks the correct serializer lock for the given test identifier, so that the next test can start. + * + * @param testIdentifier the test identifier to unlock for + */ + public void unlock(TestIdentifier testIdentifier) { + ReentrantLock lock = coverageSerializers.remove(testIdentifier.getUniqueIdObject()); + if (lock != null) { + lock.unlock(); + } + } + + /** + * Returns whether the given test identifier has a class source. + * + * @param testIdentifier the test identifier to check + * @return whether the given test identifier has a class source + */ + private boolean hasClassSource(TestIdentifier testIdentifier) { + return testIdentifier.getSource().filter(ClassSource.class::isInstance).isPresent(); + } + + /** + * Returns whether the given test identifier has a method source. + * + * @param testIdentifier the test identifier to check + * @return whether the given test identifier has a method source + */ + private boolean hasMethodSource(TestIdentifier testIdentifier) { + return testIdentifier.getSource().filter(MethodSource.class::isInstance).isPresent(); + } + + /** + * Returns whether the given test identifier is part of an atomic unit. + * + * @param testIdentifier the test identifier to check + * @return whether the given test identifier is part of an atomic unit + */ + private boolean shouldTreatAsOneUnit(TestIdentifier testIdentifier) { + return shouldTreatSpockSpecificationAsOneUnit(testIdentifier); + } + + /** + * Returns whether the test class of the given test identifier is a Spock specification + * that should be treated atomically. + * + * @param testIdentifier the test identifier to check + * @return whether the test class of the given test identifier is a Spock specification that should be treated atomically + */ + private boolean shouldTreatSpockSpecificationAsOneUnit(TestIdentifier testIdentifier) { + Optional> optionalTestClass = getTestClass(testIdentifier); + if (!optionalTestClass.isPresent()) { + return false; + } + + Class testClass = optionalTestClass.get(); + if (!isSpockSpecification(testClass)) { + return false; + } + + Set methods = Reflection.allMethods(testClass); + return hasBeforeAllAnnotations(methods) + || hasBeforeClassAnnotations(methods) + || hasAfterAllAnnotations(methods) + || hasAfterClassAnnotations(methods) + || hasClassRuleAnnotations(testClass, methods) + || hasAnnotation(testClass, STEPWISE.orElseThrow(AssertionError::new)) + || hasAnnotation(methods, STEPWISE.orElseThrow(AssertionError::new)) + || hasMethodNamed(methods, "setupSpec") + || hasMethodNamed(methods, "cleanupSpec") + || hasSharedField(testClass); + } + + /** + * Returns the test class of the given test identifier, if any. + * + * @param testIdentifier the test identifier to check + * @return the test class of the given test identifier, if any + */ + private Optional> getTestClass(TestIdentifier testIdentifier) { + if (hasClassSource(testIdentifier)) { + return Optional.of( + testIdentifier + .getSource() + .map(ClassSource.class::cast) + .orElseThrow(AssertionError::new) + .getJavaClass()); + } + + if (hasMethodSource(testIdentifier)) { + return Optional.of( + testIdentifier + .getSource() + .map(MethodSource.class::cast) + .orElseThrow(AssertionError::new) + .getJavaClass()); + } + + return Optional.empty(); + } + + /** + * Returns whether the given class is a Spock specification. + * + * @param clazz the class to check + * @return whether the given class is a Spock specification + */ + private boolean isSpockSpecification(Class clazz) { + return SPECIFICATION.filter(specification -> specification.isAssignableFrom(clazz)).isPresent(); + } + + /** + * Returns whether any of the given methods has a Jupiter {@code @BeforeAll} annotation. + * + * @param methods the methods to check + * @return whether any of the given methods has a Jupiter {@code @BeforeAll} annotation + */ + private boolean hasBeforeAllAnnotations(Set methods) { + return BEFORE_ALL.filter(beforeAll -> hasAnnotation(methods, beforeAll)).isPresent(); + } + + /** + * Returns whether any of the given methods has a JUnit 4 {@code @BeforeClass} annotation. + * + * @param methods the methods to check + * @return whether any of the given methods has a JUnit 4 {@code @BeforeClass} annotation + */ + private boolean hasBeforeClassAnnotations(Set methods) { + return BEFORE_CLASS.filter(beforeClass -> hasAnnotation(methods, beforeClass)).isPresent(); + } + + /** + * Returns whether any of the given methods has a Jupiter {@code @AfterAll} annotation. + * + * @param methods the methods to check + * @return whether any of the given methods has a Jupiter {@code @BeforeClass} annotation + */ + private boolean hasAfterAllAnnotations(Set methods) { + return AFTER_ALL.filter(afterAll -> hasAnnotation(methods, afterAll)).isPresent(); + } + + /** + * Returns whether any of the given methods has a JUnit 4 {@code @AfterClass} annotation. + * + * @param methods the methods to check + * @return whether any of the given methods has a JUnit 4 {@code @AfterClass} annotation + */ + private boolean hasAfterClassAnnotations(Set methods) { + return AFTER_CLASS.filter(afterClass -> hasAnnotation(methods, afterClass)).isPresent(); + } + + /** + * Returns whether the given class or any of the given methods has a JUnit 4 {@code @ClassRule} annotation. + * + * @param clazz the class to check + * @param methods the methods to check + * @return whether the given class or any of the given methods has a JUnit 4 {@code @ClassRule} annotation + */ + private boolean hasClassRuleAnnotations(Class clazz, Set methods) { + return CLASS_RULE.filter(aClass -> hasAnnotation(methods, aClass) + || hasAnnotation(Reflection.publicFields(clazz), aClass)).isPresent(); + } + + /** + * Returns whether the given annotated element is annotated with the given annotation class. + * + * @param annotatedElement the annotated element to check + * @param annotation the class of the annotation to check for + * @return whether the given annotated element is annotated with the given annotation class + */ + private boolean hasAnnotation(AnnotatedElement annotatedElement, Class annotation) { + return annotatedElement.isAnnotationPresent(annotation); + } + + /** + * Returns whether any of the given annotated elements is annotated with the given annotation class. + * + * @param annotatedElements the annotated elements to check + * @param annotation the class of the annotation to check for + * @return whether any of the given annotated elements is annotated with the given annotation class + */ + private boolean hasAnnotation(Set annotatedElements, Class annotation) { + return FCollection.contains(annotatedElements, annotatedElement -> annotatedElement.isAnnotationPresent(annotation)); + } + + /** + * Returns whether any of the given methods has the given name. + * + * @param methods the methods to check + * @param methodName the method name to check for + * @return whether any of the given methods has the given name + */ + private boolean hasMethodNamed(Set methods, String methodName) { + return FCollection.contains(methods, havingName(methodName)); + } + + /** + * Returns a predicate that checks whether a method has the given name. + * + * @param methodName the method name to check for + * @return a predicate that checks whether a method has the given name + */ + private Predicate havingName(String methodName) { + return method -> method.getName().equals(methodName); + } + + /** + * Returns whether the given class has a Spock {@code @Shared} field. + * + * @param clazz the class to check + * @return whether the given class has a Spock {@code @Shared} field + */ + private boolean hasSharedField(Class clazz) { + return hasAnnotation(allFields(clazz), SHARED.orElseThrow(AssertionError::new)); + } + + /** + * Returns all fields of the given class and its class hierarchy. + * + * @param clazz the class to get the fields for + * @return all fields of the given class and its class hierarchy + */ + private Set allFields(Class clazz) { + final Set fields = new LinkedHashSet<>(); + if (clazz != null) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + fields.addAll(allFields(clazz.getSuperclass())); } + return fields; } } diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterAll.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterAll.groovy new file mode 100644 index 0000000..e24a0d7 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterAll.groovy @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import org.junit.jupiter.api.AfterAll +import spock.lang.Specification + +class TestSpecWithAfterAll extends Specification { + @AfterAll + static afterAll() { + } + + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterClass.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterClass.groovy new file mode 100644 index 0000000..5e13586 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithAfterClass.groovy @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import org.junit.AfterClass +import spock.lang.Specification + +class TestSpecWithAfterClass extends Specification { + @AfterClass + static afterClass() { + } + + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeAll.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeAll.groovy new file mode 100644 index 0000000..2fff53d --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeAll.groovy @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import org.junit.jupiter.api.BeforeAll +import spock.lang.Specification + +class TestSpecWithBeforeAll extends Specification { + @BeforeAll + static beforeAll() { + } + + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeClass.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeClass.groovy new file mode 100644 index 0000000..e703288 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithBeforeClass.groovy @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import org.junit.BeforeClass +import spock.lang.Specification + +class TestSpecWithBeforeClass extends Specification { + @BeforeClass + static beforeClass() { + } + + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleField.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleField.groovy new file mode 100644 index 0000000..678b430 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleField.groovy @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import org.junit.ClassRule +import spock.lang.Specification + +class TestSpecWithClassRuleField extends Specification { + @ClassRule + public static classRule + + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleMethod.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleMethod.groovy new file mode 100644 index 0000000..7833c24 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithClassRuleMethod.groovy @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import org.junit.ClassRule +import spock.lang.Specification + +class TestSpecWithClassRuleMethod extends Specification { + @ClassRule + static afterClass() { + } + + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithSetupSpecWithoutShared.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithSetupSpecWithoutShared.groovy new file mode 100644 index 0000000..23afd09 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithSetupSpecWithoutShared.groovy @@ -0,0 +1,38 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import spock.lang.Shared +import spock.lang.Specification + +class TestSpecWithSetupSpecWithoutShared extends Specification { + static fail = true + + def setupSpec() { + fail = false + } + + def aTest() { + expect: + !fail + } + + def anotherTest() { + expect: + !fail + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithShared.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithShared.groovy new file mode 100644 index 0000000..57b7644 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithShared.groovy @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import spock.lang.Shared +import spock.lang.Specification + +class TestSpecWithShared extends Specification { + @Shared + foo + + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwise.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwise.groovy new file mode 100644 index 0000000..8d27cd5 --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwise.groovy @@ -0,0 +1,31 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import spock.lang.Specification +import spock.lang.Stepwise + +@Stepwise +class TestSpecWithStepwise extends Specification { + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwiseFeature.groovy b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwiseFeature.groovy new file mode 100644 index 0000000..a94803a --- /dev/null +++ b/src/test/groovy/org/pitest/junit5/repository/TestSpecWithStepwiseFeature.groovy @@ -0,0 +1,31 @@ +/* + * Copyright 2023 Björn Kautler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.pitest.junit5.repository + +import spock.lang.Specification +import spock.lang.Stepwise + +class TestSpecWithStepwiseFeature extends Specification { + @Stepwise + def test() { + expect: + true + + where: + i << (1..2) + } +} diff --git a/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java b/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java index f439bbc..e0f5ac0 100644 --- a/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java +++ b/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java @@ -48,8 +48,14 @@ import org.pitest.junit5.repository.TestClassWithTestFactoryAnnotation; import org.pitest.junit5.repository.TestClassWithTestTemplateAnnotation; import org.pitest.junit5.repository.TestClassWithoutAnnotations; -import org.pitest.junit5.repository.TestSpecWithCleanupSpec; import org.pitest.junit5.repository.TestSpecWithAbortingFeature; +import org.pitest.junit5.repository.TestSpecWithAfterAll; +import org.pitest.junit5.repository.TestSpecWithAfterClass; +import org.pitest.junit5.repository.TestSpecWithBeforeAll; +import org.pitest.junit5.repository.TestSpecWithBeforeClass; +import org.pitest.junit5.repository.TestSpecWithClassRuleField; +import org.pitest.junit5.repository.TestSpecWithClassRuleMethod; +import org.pitest.junit5.repository.TestSpecWithCleanupSpec; import org.pitest.junit5.repository.TestSpecWithDataDrivenFeature; import org.pitest.junit5.repository.TestSpecWithFailingCleanupSpec; import org.pitest.junit5.repository.TestSpecWithFailingFeature; @@ -59,7 +65,11 @@ import org.pitest.junit5.repository.TestSpecWithMixedPassAndFail; import org.pitest.junit5.repository.TestSpecWithMultiplePassingFeatures; import org.pitest.junit5.repository.TestSpecWithSetupSpec; +import org.pitest.junit5.repository.TestSpecWithSetupSpecWithoutShared; +import org.pitest.junit5.repository.TestSpecWithShared; import org.pitest.junit5.repository.TestSpecWithSimpleFeature; +import org.pitest.junit5.repository.TestSpecWithStepwise; +import org.pitest.junit5.repository.TestSpecWithStepwiseFeature; import org.pitest.junit5.repository.TestSpecWithTags; import org.pitest.junit5.repository.TestSpecWithoutFeatures; import org.pitest.testapi.Description; @@ -321,8 +331,8 @@ void findsAndRunsTestsWithAfterAll() { } @Test - void findsAndRunsTestsWithCleanupSpec() { - findsAndRunsNTests(2, TestSpecWithCleanupSpec.class); + void findsAndRunsAtomicTestWithCleanupSpec() { + findsAndRunsNTests(1, TestSpecWithCleanupSpec.class); } @Test @@ -331,8 +341,8 @@ void findsAndRunsTestsWithBeforeAll() { } @Test - void findsAndRunsTestsWithSetupSpec() { - findsAndRunsNTests(2, TestSpecWithSetupSpec.class); + void findsAndRunsAtomicTestWithSetupSpec() { + findsAndRunsNTests(1, TestSpecWithSetupSpec.class); } @Test @@ -342,8 +352,8 @@ void findsAndRunsTestsWithFailingAfterAll() { } @Test - void findsAndRunsTestsWithFailingCleanupSpec() { - findsAndRunsNTests(2, TestSpecWithFailingCleanupSpec.class); + void findsAndRunsAtomicTestWithFailingCleanupSpec() { + findsAndRunsNTests(1, TestSpecWithFailingCleanupSpec.class); } @Test @@ -352,8 +362,8 @@ void findsNoTestsWithFailingBeforeAll() { } @Test - void findsNoTestsWithFailingSetupSpec() { - findsAndRunsNTests(0, TestSpecWithFailingSetupSpec.class); + void findsAndRunsAtomicTestWithFailingSetupSpec() { + findsAndRunsNTests(1, TestSpecWithFailingSetupSpec.class); } @Test @@ -361,6 +371,56 @@ void findsNoTestsWithNestedTestClassWithoutAnnotations() { findsAndRunsNTests(0, TestClassWithNestedClassWithoutAnnotations.class); } + @Test + void findsAndRunsAtomicTestWithAfterAll() { + findsAndRunsNTests(1, TestSpecWithAfterAll.class); + } + + @Test + void findsAndRunsAtomicTestWithAfterClass() { + findsAndRunsNTests(1, TestSpecWithAfterClass.class); + } + + @Test + void findsAndRunsAtomicTestWithBeforeAll() { + findsAndRunsNTests(1, TestSpecWithBeforeAll.class); + } + + @Test + void findsAndRunsAtomicTestWithBeforeClass() { + findsAndRunsNTests(1, TestSpecWithBeforeClass.class); + } + + @Test + void findsAndRunsAtomicTestWithClassRuleField() { + findsAndRunsNTests(1, TestSpecWithClassRuleField.class); + } + + @Test + void findsAndRunsAtomicTestWithClassRuleMethod() { + findsAndRunsNTests(1, TestSpecWithClassRuleMethod.class); + } + + @Test + void findsAndRunsAtomicTestWithSetupSpecWithoutShared() { + findsAndRunsNTests(1, TestSpecWithSetupSpecWithoutShared.class); + } + + @Test + void findsAndRunsAtomicTestWithShared() { + findsAndRunsNTests(1, TestSpecWithShared.class); + } + + @Test + void findsAndRunsAtomicTestWithStepwise() { + findsAndRunsNTests(1, TestSpecWithStepwise.class); + } + + @Test + void findsAndRunsAtomicTestWithStepwiseFeature() { + findsAndRunsNTests(1, TestSpecWithStepwiseFeature.class); + } + @Test void findsAndRunsCucumberTests() { findsAndRunsNTests(1, RunCucumberTest.class);