Introduce PHPBench and move performance tests from AnalyserIntegrationTest to RegressionBench

.github/workflows/bench.yml

@@ -0,0 +1,95 @@
+name: "Benchmark"
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'compiler/**'
+      - 'apigen/**'
+      - 'changelog-generator/**'
+      - 'issue-bot/**'
+  push:
+    branches:
+      - "2.1.x"
+    paths-ignore:
+      - 'compiler/**'
+      - 'apigen/**'
+      - 'changelog-generator/**'
+      - 'issue-bot/**'
+
+concurrency:
+  group: bench-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  baseline:
+    name: "Baseline"
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2
+        with:
+          egress-policy: audit
+
+      - name: "Checkout base branch"
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1" # v2
+        with:
+          coverage: "none"
+          php-version: "8.5"
+          tools: pecl
+          extensions: ds,mbstring
+          ini-file: development
+          ini-values: memory_limit=-1
+
+      - uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # v3
+      - uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # v3
+        with:
+          working-directory: "tests/"
+
+      - name: "Run phpbench baseline"
+        run: "tests/vendor/bin/phpbench run --dump-file=tests/bench/storage/baseline.xml"
+
+      - name: "Upload baseline artifact"
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: phpbench-baseline
+          path: tests/bench/storage/baseline.xml
+
+  test:
+    name: "Test"
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2
+        with:
+          egress-policy: audit
+
+      - name: "Checkout"
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1" # v2
+        with:
+          coverage: "none"
+          php-version: "8.5"
+          tools: pecl
+          extensions: ds,mbstring
+          ini-file: development
+          ini-values: memory_limit=-1
+
+      - uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # v3
+      - uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # v3
+        with:
+          working-directory: "tests/"
+
+      - name: "Run phpbench test"
+        run: "tests/vendor/bin/phpbench run --file=tests/bench/storage/baseline.xml --report=aggregate"

.github/workflows/tests.yml

@@ -259,7 +259,7 @@ jobs:
 
       - name: "Downgrade PHPUnit with Paratest"
         shell: bash
-        run: "composer require --dev phpunit/phpunit:^9.6 brianium/paratest:^6.5 symfony/console:^5.4 symfony/process:^5.4 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs --working-dir=tests"
+        run: "composer require --dev phpbench/phpbench:^1.2.15 phpunit/phpunit:^9.6 brianium/paratest:^6.5 symfony/console:^5.4 symfony/process:^5.4 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs --working-dir=tests"
 
       - uses: ./.github/actions/downgrade-code
         with:

.gitignore

@@ -13,3 +13,4 @@
 /tests/PHPStan/Reflection/data/golden/
 tmp/.memory_limit
 e2e/bashunit
+/.phpbench

Makefile

@@ -14,6 +14,8 @@ tests-golden-reflection:
 lint:
 	XDEBUG_MODE=off php vendor/bin/parallel-lint --colors \
 		--exclude tests/PHPStan/Analyser/data \
+		--exclude tests/bench/data \
+		--exclude tests/bench/RegressionBench.php \
 		--exclude tests/PHPStan/Analyser/nsrt \
 		--exclude tests/PHPStan/Rules/Methods/data \
 		--exclude tests/PHPStan/Rules/Functions/data \

build/phpstan.neon

@@ -31,6 +31,7 @@ parameters:
 		- ../tests/PHPStan/Analyser/nsrt/*
 		- ../tests/PHPStan/Analyser/traits/*
 		- ../tests/notAutoloaded/*
+		- ../tests/bench/*
 		- ../tests/PHPStan/Reflection/UnionTypesTest.php
 		- ../tests/PHPStan/Reflection/MixedTypeTest.php
 		- ../tests/e2e/magic-setter/*

composer.json

@@ -153,7 +153,8 @@
 		},
 		"classmap": [
 			"tests/e2e",
-			"tests/PHPStan"
+			"tests/PHPStan",
+			"tests/bench"
 		]
 	},
 	"repositories": [

phpbench.json

@@ -0,0 +1,7 @@
+{
+	"$schema":"./tests/vendor/phpbench/phpbench/phpbench.schema.json",
+	"runner.bootstrap": "vendor/autoload.php",
+	"runner.path": "tests/bench",
+	"runner.file_pattern": "*Bench.php",
+	"storage.xml_storage_path": "tests/bench/storage"
+}

phpcs.xml

@@ -53,7 +53,7 @@
 	</rule>
 	<rule ref="SlevomatCodingStandard.Functions.UnusedInheritedVariablePassedToClosure.UnusedInheritedVariable">
 		<exclude-pattern>src/Command/CommandHelper.php</exclude-pattern>
-		<exclude-pattern>src/Testing/PHPStanTestCase.php</exclude-pattern>
+		<exclude-pattern>src/Testing/PHPStanTestCaseTrait.php</exclude-pattern>
 	</rule>
 	<rule ref="SlevomatCodingStandard.Exceptions.ReferenceThrowableOnly.ReferencedGeneralException">
 		<exclude-pattern>tests</exclude-pattern>
@@ -65,6 +65,7 @@
 				<element key="compiler/tests" value="PHPStan\Compiler"/>
 				<element key="src" value="PHPStan"/>
 				<element key="tests/PHPStan" value="PHPStan"/>
+				<element key="tests/bench" value="PHPStan\Benchmark"/>
 				<element key="tests/e2e" value="PHPStan\Tests"/>
 				<element key="apigen/src" value="PHPStan\ApiGen"/>
 				<element key="changelog-generator/src" value="PHPStan\ChangelogGenerator"/>

src/Testing/PHPStanTestCase.php

@@ -8,14 +8,9 @@
 use PHPStan\Analyser\ScopeFactory;
 use PHPStan\Analyser\TypeSpecifier;
 use PHPStan\BetterReflection\Reflector\Reflector;
-use PHPStan\DependencyInjection\Container;
-use PHPStan\DependencyInjection\ContainerFactory;
 use PHPStan\DependencyInjection\Reflection\ClassReflectionExtensionRegistryProvider;
 use PHPStan\DependencyInjection\Type\ExpressionTypeResolverExtensionRegistryProvider;
 use PHPStan\DependencyInjection\Type\OperatorTypeSpecifyingExtensionRegistryProvider;
-use PHPStan\File\FileHelper;
-use PHPStan\Internal\DirectoryCreator;
-use PHPStan\Internal\DirectoryCreatorException;
 use PHPStan\Node\Printer\ExprPrinter;
 use PHPStan\Parser\Parser;
 use PHPStan\Php\ComposerPhpVersionFactory;
@@ -32,76 +27,17 @@
 use PHPStan\Type\UsefulTypeAliasResolver;
 use PHPUnit\Framework\ExpectationFailedException;
 use PHPUnit\Framework\TestCase;
-use function array_merge;
 use function count;
-use function hash;
 use function implode;
 use function rtrim;
 use function sprintf;
-use function sys_get_temp_dir;
 use const DIRECTORY_SEPARATOR;
-use const PHP_VERSION_ID;
 
 /** @api */
 abstract class PHPStanTestCase extends TestCase
 {
 
-	/** @var array<string, Container> */
-	private static array $containers = [];
-
-	/** @api */
-	public static function getContainer(): Container
-	{
-		$additionalConfigFiles = [__DIR__ . '/TestCase.neon'];
-		foreach (static::getAdditionalConfigFiles() as $configFile) {
-			$additionalConfigFiles[] = $configFile;
-		}
-		$cacheKey = hash('sha256', implode("\n", $additionalConfigFiles));
-
-		if (!isset(self::$containers[$cacheKey])) {
-			$tmpDir = sys_get_temp_dir() . '/phpstan-tests';
-			try {
-				DirectoryCreator::ensureDirectoryExists($tmpDir, 0777);
-			} catch (DirectoryCreatorException $e) {
-				self::fail($e->getMessage());
-			}
-
-			$rootDir = __DIR__ . '/../..';
-			$fileHelper = new FileHelper($rootDir);
-			$rootDir = $fileHelper->normalizePath($rootDir, '/');
-			$containerFactory = new ContainerFactory($rootDir);
-			$container = $containerFactory->create($tmpDir, array_merge([
-				$containerFactory->getConfigDirectory() . '/config.level8.neon',
-			], $additionalConfigFiles), []);
-			self::$containers[$cacheKey] = $container;
-
-			foreach ($container->getParameter('bootstrapFiles') as $bootstrapFile) {
-				(static function (string $file) use ($container): void {
-					require_once $file;
-				})($bootstrapFile);
-			}
-
-			if (PHP_VERSION_ID >= 80000) {
-				require_once __DIR__ . '/../../stubs/runtime/Enum/UnitEnum.php';
-				require_once __DIR__ . '/../../stubs/runtime/Enum/BackedEnum.php';
-				require_once __DIR__ . '/../../stubs/runtime/Enum/ReflectionEnum.php';
-				require_once __DIR__ . '/../../stubs/runtime/Enum/ReflectionEnumUnitCase.php';
-				require_once __DIR__ . '/../../stubs/runtime/Enum/ReflectionEnumBackedCase.php';
-			}
-		} else {
-			ContainerFactory::postInitializeContainer(self::$containers[$cacheKey]);
-		}
-
-		return self::$containers[$cacheKey];
-	}
-
-	/**
-	 * @return string[]
-	 */
-	public static function getAdditionalConfigFiles(): array
-	{
-		return [];
-	}
+	use PHPStanTestCaseTrait;
 
 	public static function getParser(): Parser
 	{
@@ -188,11 +124,6 @@ protected function shouldTreatPhpDocTypesAsCertain(): bool
 		return true;
 	}
 
-	public static function getFileHelper(): FileHelper
-	{
-		return self::getContainer()->getByType(FileHelper::class);
-	}
-
 	/**
 	 * Provides a DIRECTORY_SEPARATOR agnostic assertion helper, to compare file paths.
 	 *

src/Testing/PHPStanTestCaseTrait.php

@@ -0,0 +1,81 @@
+<?php declare(strict_types = 1);
+
+namespace PHPStan\Testing;
+
+use PHPStan\DependencyInjection\Container;
+use PHPStan\DependencyInjection\ContainerFactory;
+use PHPStan\File\FileHelper;
+use PHPStan\Internal\DirectoryCreator;
+use PHPStan\Internal\DirectoryCreatorException;
+use RuntimeException;
+use function array_merge;
+use function hash;
+use function implode;
+use function sys_get_temp_dir;
+use const PHP_VERSION_ID;
+
+trait PHPStanTestCaseTrait
+{
+
+	/** @var array<string, Container> */
+	private static array $containers = [];
+
+	public static function getContainer(): Container
+	{
+		$additionalConfigFiles = [__DIR__ . '/TestCase.neon'];
+		foreach (static::getAdditionalConfigFiles() as $configFile) {
+			$additionalConfigFiles[] = $configFile;
+		}
+		$cacheKey = hash('sha256', implode("\n", $additionalConfigFiles));
+
+		if (!isset(self::$containers[$cacheKey])) {
+			$tmpDir = sys_get_temp_dir() . '/phpstan-tests';
+			try {
+				DirectoryCreator::ensureDirectoryExists($tmpDir, 0777);
+			} catch (DirectoryCreatorException $e) {
+				throw new RuntimeException($e->getMessage(), previous: $e);
+			}
+
+			$rootDir = __DIR__ . '/../..';
+			$fileHelper = new FileHelper($rootDir);
+			$rootDir = $fileHelper->normalizePath($rootDir, '/');
+			$containerFactory = new ContainerFactory($rootDir);
+			$container = $containerFactory->create($tmpDir, array_merge([
+				$containerFactory->getConfigDirectory() . '/config.level8.neon',
+			], $additionalConfigFiles), []);
+			self::$containers[$cacheKey] = $container;
+
+			foreach ($container->getParameter('bootstrapFiles') as $bootstrapFile) {
+				(static function (string $file) use ($container): void {
+					require_once $file;
+				})($bootstrapFile);
+			}
+
+			if (PHP_VERSION_ID >= 80000) {
+				require_once __DIR__ . '/../../stubs/runtime/Enum/UnitEnum.php';
+				require_once __DIR__ . '/../../stubs/runtime/Enum/BackedEnum.php';
+				require_once __DIR__ . '/../../stubs/runtime/Enum/ReflectionEnum.php';
+				require_once __DIR__ . '/../../stubs/runtime/Enum/ReflectionEnumUnitCase.php';
+				require_once __DIR__ . '/../../stubs/runtime/Enum/ReflectionEnumBackedCase.php';
+			}
+		} else {
+			ContainerFactory::postInitializeContainer(self::$containers[$cacheKey]);
+		}
+
+		return self::$containers[$cacheKey];
+	}
+
+	/**
+	 * @return string[]
+	 */
+	public static function getAdditionalConfigFiles(): array
+	{
+		return [];
+	}
+
+	public static function getFileHelper(): FileHelper
+	{
+		return self::getContainer()->getByType(FileHelper::class);
+	}
+
+}