Fix: Use separate PHPT tests for lowest and highest versions of phpunit/phpunit

Author: localheinz

.github/workflows/integrate.yaml

@@ -762,35 +762,35 @@ jobs:
 
       - name: "Run end-to-end tests with phpunit/phpunit:6.5.0"
         if: "matrix.phpunit-version == '6.5.0'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version06/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version06/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Run end-to-end tests with phpunit/phpunit:7.5.0"
         if: "matrix.phpunit-version == '7.5.0'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version07/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version07/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Run end-to-end tests with phpunit/phpunit:8.5.19"
         if: "matrix.phpunit-version == '8.5.19'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version08/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version08/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Run end-to-end tests with phpunit/phpunit:8.5.40"
         if: "matrix.phpunit-version == '8.5.40'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version08/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version08/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Run end-to-end tests with phpunit/phpunit:9.0.0"
         if: "matrix.phpunit-version == '9.0.0'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version09/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version09/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Run end-to-end tests with phpunit/phpunit:10.0.0"
         if: "matrix.phpunit-version == '10.0.0'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version10/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version10/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Run end-to-end tests with phpunit/phpunit:11.0.0"
         if: "matrix.phpunit-version == '11.0.0'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version11/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version11/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Run end-to-end tests with phpunit/phpunit:12.0.0"
         if: "matrix.phpunit-version == '12.0.0'"
-        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version12/phpunit.xml"
+        run: "vendor/bin/phpunit --colors=always --configuration=test/EndToEnd/Version12/phpunit.xml --testsuite=${{ matrix.dependencies }}"
 
       - name: "Download Phar"
         uses: "actions/[email protected]"

Makefile

@@ -54,15 +54,15 @@ static-code-analysis-baseline: vendor ## Generates a baseline for static code an
 tests: phar ## Runs unit, end-to-end, and phar tests with phpunit/phpunit on PHP 7.4
 	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^6.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Unit/phpunit.xml; git checkout HEAD -- composer.json composer.lock
 
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^6.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version06/phpunit.xml; git checkout HEAD -- composer.json composer.lock
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^7.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version07/phpunit.xml; git checkout HEAD -- composer.json composer.lock
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^8.5.19 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version08/phpunit.xml; git checkout HEAD -- composer.json composer.lock
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^9.0.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version09/phpunit.xml; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^6.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version06/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^7.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version07/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^8.5.19 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version08/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^9.0.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/EndToEnd/Version09/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
 
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^6.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version06/phpunit.xml; git checkout HEAD -- composer.json composer.lock
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^7.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version07/phpunit.xml; git checkout HEAD -- composer.json composer.lock
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^8.5.19 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version08/phpunit.xml; git checkout HEAD -- composer.json composer.lock
-	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^9.0.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version09/phpunit.xml; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^6.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version06/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^7.5.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version07/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^8.5.19 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version08/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
+	composer config platform.php --unset && composer remove ergebnis/php-cs-fixer-config phpstan/phpstan-phpunit --dev --no-interaction --quiet && composer require phpunit/phpunit:^9.0.0 --no-interaction --quiet --update-with-all-dependencies && vendor/bin/phpunit --configuration=test/Phar/Version09/phpunit.xml --test-suite=highest; git checkout HEAD -- composer.json composer.lock
 
 vendor: composer.json composer.lock
 	composer validate --strict

test/EndToEnd/Version06/TestCase/Bare/test.phpt renamed to test/EndToEnd/Version06/TestCase/Bare/test-highest.phpt

@@ -0,0 +1,30 @@
+--TEST--
+With a test case that does not sleep in methods that are not test methods
+--FILE--
+<?php
+
+declare(strict_types=1);
+
+use PHPUnit\TextUI;
+
+$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version06/TestCase/Bare/phpunit.xml';
+
+require_once __DIR__ . '/../../../../../vendor/autoload.php';
+
+PHPUnit\TextUI\Command::main();
+--EXPECTF--
+PHPUnit %s
+
+Runtime: %s
+Configuration: %s/EndToEnd/Version06/TestCase/Bare/phpunit.xml
+
+...                                                                 3 / 3 (100%)
+
+Detected 2 tests where the duration exceeded the maximum duration.
+
+1. 00:00.3%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\Bare\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #1
+2. 00:00.2%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\Bare\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #0
+
+Time: %s, Memory: %s
+
+OK (3 tests, 3 assertions)

test/EndToEnd/Version06/TestCase/Bare/test-lowest.phpt

@@ -0,0 +1,31 @@
+--TEST--
+With a test case that sleeps in data provider, hook, and test methods
+--FILE--
+<?php
+
+declare(strict_types=1);
+
+use PHPUnit\TextUI;
+
+$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version06/TestCase/Combination/phpunit.xml';
+
+require_once __DIR__ . '/../../../../../vendor/autoload.php';
+
+PHPUnit\TextUI\Command::main();
+--EXPECTF--
+PHPUnit %s
+
+Runtime: %s
+Configuration: %s/EndToEnd/Version06/TestCase/Combination/phpunit.xml
+
+...                                                                 3 / 3 (100%)
+
+Detected 3 tests where the duration exceeded the maximum duration.
+
+1. 00:00.9%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\Combination\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #1
+2. 00:00.8%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\Combination\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #0
+3. 00:00.6%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\Combination\SleeperTest::testSleeperSleepsLessThanMaximumDurationFromXmlConfiguration
+
+Time: %s, Memory: %s
+
+OK (3 tests, 3 assertions)

test/EndToEnd/Version06/TestCase/Combination/test.phpt renamed to test/EndToEnd/Version06/TestCase/Combination/test-highest.phpt

@@ -0,0 +1,31 @@
+--TEST--
+With a test case that sleeps in a method with @after annotation
+--FILE--
+<?php
+
+declare(strict_types=1);
+
+use PHPUnit\TextUI;
+
+$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version06/TestCase/WithAfterAnnotation/phpunit.xml';
+
+require_once __DIR__ . '/../../../../../vendor/autoload.php';
+
+PHPUnit\TextUI\Command::main();
+--EXPECTF--
+PHPUnit %s
+
+Runtime: %s
+Configuration: %s/EndToEnd/Version06/TestCase/WithAfterAnnotation/phpunit.xml
+
+...                                                                 3 / 3 (100%)
+
+Detected 3 tests where the duration exceeded the maximum duration.
+
+1. 00:00.4%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\WithAfterAnnotation\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #1
+2. 00:00.3%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\WithAfterAnnotation\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #0
+3. 00:00.1%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\WithAfterAnnotation\SleeperTest::testSleeperSleepsLessThanMaximumDurationFromXmlConfiguration
+
+Time: %s, Memory: %s
+
+OK (3 tests, 3 assertions)

test/EndToEnd/Version06/TestCase/Combination/test-lowest.phpt

@@ -0,0 +1,30 @@
+--TEST--
+With a test case that sleeps in a method with @afterClass annotation
+--FILE--
+<?php
+
+declare(strict_types=1);
+
+use PHPUnit\TextUI;
+
+$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version06/TestCase/WithAfterClassAnnotation/phpunit.xml';
+
+require_once __DIR__ . '/../../../../../vendor/autoload.php';
+
+PHPUnit\TextUI\Command::main();
+--EXPECTF--
+PHPUnit %s
+
+Runtime: %s
+Configuration: %s/EndToEnd/Version06/TestCase/WithAfterClassAnnotation/phpunit.xml
+
+...                                                                 3 / 3 (100%)
+
+Detected 2 tests where the duration exceeded the maximum duration.
+
+1. 00:00.3%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\WithAfterClassAnnotation\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #1
+2. 00:00.2%s (00:00.100) Ergebnis\PHPUnit\SlowTestDetector\Test\EndToEnd\Version06\TestCase\WithAfterClassAnnotation\SleeperTest::testSleeperSleepsLongerThanMaximumDurationFromXmlConfigurationWithDataProvider with data set #0
+
+Time: %s, Memory: %s
+
+OK (3 tests, 3 assertions)