Skip to content

Collected data: reduce memory consumption & result cache size #3879

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 18, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/Analyser/Analyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
namespace PHPStan\Analyser;

use Closure;
use PHPStan\Collectors\CollectedData;
use PhpParser\Node;
use PHPStan\Collectors\Collector;
use PHPStan\Collectors\Registry as CollectorRegistry;
use PHPStan\Rules\Registry as RuleRegistry;
use Throwable;
Expand Down Expand Up @@ -59,7 +60,7 @@ public function analyse(
$linesToIgnore = [];
$unmatchedLineIgnores = [];

/** @var list<CollectedData> $collectedData */
/** @var array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData */
$collectedData = [];

$internalErrorsCount = 0;
Expand Down
7 changes: 4 additions & 3 deletions src/Analyser/AnalyserResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

namespace PHPStan\Analyser;

use PHPStan\Collectors\CollectedData;
use PhpParser\Node;
use PHPStan\Collectors\Collector;
use PHPStan\Dependency\RootExportedNode;
use function usort;

Expand All @@ -22,7 +23,7 @@ final class AnalyserResult
* @param list<Error> $locallyIgnoredErrors
* @param array<string, LinesToIgnore> $linesToIgnore
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
* @param list<CollectedData> $collectedData
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
* @param list<InternalError> $internalErrors
* @param array<string, array<string>>|null $dependencies
* @param array<string, array<RootExportedNode>> $exportedNodes
Expand Down Expand Up @@ -125,7 +126,7 @@ public function getInternalErrors(): array
}

/**
* @return list<CollectedData>
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
*/
public function getCollectedData(): array
{
Expand Down
10 changes: 3 additions & 7 deletions src/Analyser/FileAnalyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
use PHPStan\BetterReflection\NodeCompiler\Exception\UnableToCompileNode;
use PHPStan\BetterReflection\Reflection\Exception\CircularReference;
use PHPStan\BetterReflection\Reflector\Exception\IdentifierNotFound;
use PHPStan\Collectors\CollectedData;
use PHPStan\Collectors\Collector;
use PHPStan\Collectors\Registry as CollectorRegistry;
use PHPStan\Dependency\DependencyResolver;
use PHPStan\Node\FileNode;
Expand Down Expand Up @@ -76,7 +76,7 @@ public function analyseFile(
/** @var list<Error> $locallyIgnoredErrors */
$locallyIgnoredErrors = [];

/** @var list<CollectedData> $fileCollectedData */
/** @var array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $fileCollectedData */
$fileCollectedData = [];

$fileDependencies = [];
Expand Down Expand Up @@ -195,11 +195,7 @@ public function analyseFile(
continue;
}

$fileCollectedData[] = new CollectedData(
$collectedData,
$scope->getFile(),
get_class($collector),
);
$fileCollectedData[$scope->getFile()][get_class($collector)][] = $collectedData;
}

try {
Expand Down
7 changes: 4 additions & 3 deletions src/Analyser/FileAnalyserResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

namespace PHPStan\Analyser;

use PHPStan\Collectors\CollectedData;
use PhpParser\Node;
use PHPStan\Collectors\Collector;
use PHPStan\Dependency\RootExportedNode;

/**
Expand All @@ -16,7 +17,7 @@ final class FileAnalyserResult
* @param list<Error> $filteredPhpErrors
* @param list<Error> $allPhpErrors
* @param list<Error> $locallyIgnoredErrors
* @param list<CollectedData> $collectedData
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
* @param list<string> $dependencies
* @param list<RootExportedNode> $exportedNodes
* @param LinesToIgnore $linesToIgnore
Expand Down Expand Up @@ -69,7 +70,7 @@ public function getLocallyIgnoredErrors(): array
}

/**
* @return list<CollectedData>
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
*/
public function getCollectedData(): array
{
Expand Down
7 changes: 4 additions & 3 deletions src/Analyser/ResultCache/ResultCache.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@

namespace PHPStan\Analyser\ResultCache;

use PhpParser\Node;
use PHPStan\Analyser\Error;
use PHPStan\Analyser\FileAnalyserResult;
use PHPStan\Collectors\CollectedData;
use PHPStan\Collectors\Collector;
use PHPStan\Dependency\RootExportedNode;

/**
Expand All @@ -20,7 +21,7 @@ final class ResultCache
* @param array<string, list<Error>> $locallyIgnoredErrors
* @param array<string, LinesToIgnore> $linesToIgnore
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
* @param array<string, array<CollectedData>> $collectedData
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
* @param array<string, array<string>> $dependencies
* @param array<string, array<RootExportedNode>> $exportedNodes
* @param array<string, array{string, bool, string}> $projectExtensionFiles
Expand Down Expand Up @@ -101,7 +102,7 @@ public function getUnmatchedLineIgnores(): array
}

/**
* @return array<string, array<CollectedData>>
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
*/
public function getCollectedData(): array
{
Expand Down
26 changes: 11 additions & 15 deletions src/Analyser/ResultCache/ResultCacheManager.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
namespace PHPStan\Analyser\ResultCache;

use Nette\Neon\Neon;
use PhpParser\Node;
use PHPStan\Analyser\AnalyserResult;
use PHPStan\Analyser\Error;
use PHPStan\Analyser\FileAnalyserResult;
use PHPStan\Collectors\CollectedData;
use PHPStan\Collectors\Collector;
use PHPStan\Command\Output;
use PHPStan\Dependency\ExportedNodeFetcher;
use PHPStan\Dependency\RootExportedNode;
Expand Down Expand Up @@ -406,10 +408,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
$freshLocallyIgnoredErrorsByFile[$error->getFilePath()][] = $error;
}

$freshCollectedDataByFile = [];
foreach ($analyserResult->getCollectedData() as $collectedData) {
$freshCollectedDataByFile[$collectedData->getFilePath()][] = $collectedData;
}
$freshCollectedDataByFile = $analyserResult->getCollectedData();

$meta = $resultCache->getMeta();
$projectConfigArray = $meta['projectConfig'];
Expand Down Expand Up @@ -524,13 +523,6 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
}
}

$flatCollectedData = [];
foreach ($collectedDataByFile as $fileCollectedData) {
foreach ($fileCollectedData as $collectedData) {
$flatCollectedData[] = $collectedData;
}
}

return new ResultCacheProcessResult(new AnalyserResult(
$flatErrors,
$analyserResult->getFilteredPhpErrors(),
Expand All @@ -539,7 +531,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
$linesToIgnore,
$unmatchedLineIgnores,
$internalErrors,
$flatCollectedData,
$collectedDataByFile,
$dependencies,
$exportedNodes,
$analyserResult->hasReachedInternalErrorsCountLimit(),
Expand Down Expand Up @@ -584,8 +576,8 @@ private function mergeLocallyIgnoredErrors(ResultCache $resultCache, array $fres
}

/**
* @param array<string, array<CollectedData>> $freshCollectedDataByFile
* @return array<string, array<CollectedData>>
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $freshCollectedDataByFile
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
*/
private function mergeCollectedData(ResultCache $resultCache, array $freshCollectedDataByFile): array
{
Expand Down Expand Up @@ -704,7 +696,7 @@ private function mergeUnmatchedLineIgnores(ResultCache $resultCache, array $fres
* @param array<string, list<Error>> $locallyIgnoredErrors
* @param array<string, LinesToIgnore> $linesToIgnore
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
* @param array<string, array<CollectedData>> $collectedData
* @param array<string, array<string, list<CollectedData>>> $collectedData
* @param array<string, array<string>> $dependencies
* @param array<string, array<RootExportedNode>> $exportedNodes
* @param array<string, array{string, bool, string}> $projectExtensionFiles
Expand Down Expand Up @@ -760,6 +752,10 @@ private function save(
ksort($collectedData);
ksort($invertedDependencies);

foreach ($collectedData as & $collectedDataPerFile) {
ksort($collectedDataPerFile);
}

foreach ($invertedDependencies as $file => $fileData) {
$dependentFiles = $fileData['dependentFiles'];
sort($dependentFiles);
Expand Down
21 changes: 20 additions & 1 deletion src/Command/AnalyseApplication.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

namespace PHPStan\Command;

use PhpParser\Node;
use PHPStan\Analyser\AnalyserResult;
use PHPStan\Analyser\AnalyserResultFinalizer;
use PHPStan\Analyser\Ignore\IgnoredErrorHelper;
use PHPStan\Analyser\ResultCache\ResultCacheManagerFactory;
use PHPStan\Collectors\CollectedData;
use PHPStan\Collectors\Collector;
use PHPStan\Internal\BytesHelper;
use PHPStan\PhpDoc\StubFilesProvider;
use PHPStan\PhpDoc\StubValidator;
Expand Down Expand Up @@ -150,7 +153,7 @@ public function analyse(
$notFileSpecificErrors,
$internalErrors,
[],
$collectedData,
$this->mapCollectedData($collectedData),
$defaultLevelUsed,
$projectConfigFile,
$savedResultCache,
Expand All @@ -160,6 +163,22 @@ public function analyse(
);
}

/**
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
*
* @return list<CollectedData>
*/
private function mapCollectedData(array $collectedData): array
{
$result = [];
foreach ($collectedData as $file => $dataPerCollector) {
foreach ($dataPerCollector as $collectorType => $rawData) {
$result[] = new CollectedData($rawData, $file, $collectorType);
}
}
return $result;
}

/**
* @param string[] $files
* @param string[] $allAnalysedFiles
Expand Down
8 changes: 6 additions & 2 deletions src/Command/WorkerCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,12 @@ private function runWorker(
foreach ($fileAnalyserResult->getLocallyIgnoredErrors() as $locallyIgnoredError) {
$locallyIgnoredErrors[] = $locallyIgnoredError;
}
foreach ($fileAnalyserResult->getCollectedData() as $data) {
$collectedData[] = $data;
foreach ($fileAnalyserResult->getCollectedData() as $collectedFile => $dataPerCollector) {
foreach ($dataPerCollector as $collectorType => $collectorData) {
foreach ($collectorData as $data) {
$collectedData[$collectedFile][$collectorType][] = $data;
}
}
}
} catch (Throwable $t) {
$internalErrorsCount++;
Expand Down
15 changes: 5 additions & 10 deletions src/Node/CollectedDataNode.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@

use PhpParser\Node;
use PhpParser\NodeAbstract;
use PHPStan\Collectors\CollectedData;
use PHPStan\Collectors\Collector;
use function array_key_exists;

/**
* @api
Expand All @@ -15,7 +13,7 @@ final class CollectedDataNode extends NodeAbstract implements VirtualNode
{

/**
* @param CollectedData[] $collectedData
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
*/
public function __construct(private array $collectedData, private bool $onlyFiles)
{
Expand All @@ -31,17 +29,14 @@ public function __construct(private array $collectedData, private bool $onlyFile
public function get(string $collectorType): array
{
$result = [];
foreach ($this->collectedData as $collectedData) {
if ($collectedData->getCollectorType() !== $collectorType) {
foreach ($this->collectedData as $filePath => $collectedDataPerCollector) {
if (!isset($collectedDataPerCollector[$collectorType])) {
continue;
}

$filePath = $collectedData->getFilePath();
if (!array_key_exists($filePath, $result)) {
$result[$filePath] = [];
foreach ($collectedDataPerCollector[$collectorType] as $rawData) {
$result[$filePath][] = $rawData;
}

$result[$filePath][] = $collectedData->getData();
}

return $result;
Expand Down
9 changes: 6 additions & 3 deletions src/Parallel/ParallelAnalyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
use PHPStan\Analyser\AnalyserResult;
use PHPStan\Analyser\Error;
use PHPStan\Analyser\InternalError;
use PHPStan\Collectors\CollectedData;
use PHPStan\Dependency\RootExportedNode;
use PHPStan\Process\ProcessHelper;
use React\EventLoop\LoopInterface;
Expand Down Expand Up @@ -211,8 +210,12 @@ public function analyse(
$locallyIgnoredErrors[] = $locallyIgnoredFileError;
}

foreach ($json['collectedData'] as $jsonData) {
$collectedData[] = CollectedData::decode($jsonData);
foreach ($json['collectedData'] as $file => $jsonDataByCollector) {
foreach ($jsonDataByCollector as $collectorType => $listOfCollectedData) {
foreach ($listOfCollectedData as $rawCollectedData) {
$collectedData[$file][$collectorType][] = $rawCollectedData;
}
}
}

/**
Expand Down
Loading