Skip to content

Collected data: reduce memory consumption & result cache size #3879

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 18, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/Analyser/Analyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
use function count;
use function memory_get_peak_usage;

/**
* @phpstan-import-type CollectorData from CollectedData
*/
final class Analyser
{

Expand Down Expand Up @@ -59,7 +62,7 @@ public function analyse(
$linesToIgnore = [];
$unmatchedLineIgnores = [];

/** @var list<CollectedData> $collectedData */
/** @var CollectorData $collectedData */
$collectedData = [];

$internalErrorsCount = 0;
Expand Down
5 changes: 3 additions & 2 deletions src/Analyser/AnalyserResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

/**
* @phpstan-import-type LinesToIgnore from FileAnalyserResult
* @phpstan-import-type CollectorData from CollectedData
*/
final class AnalyserResult
{
Expand All @@ -22,7 +23,7 @@ final class AnalyserResult
* @param list<Error> $locallyIgnoredErrors
* @param array<string, LinesToIgnore> $linesToIgnore
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
* @param list<CollectedData> $collectedData
* @param CollectorData $collectedData
* @param list<InternalError> $internalErrors
* @param array<string, array<string>>|null $dependencies
* @param array<string, array<RootExportedNode>> $exportedNodes
Expand Down Expand Up @@ -125,7 +126,7 @@ public function getInternalErrors(): array
}

/**
* @return list<CollectedData>
* @return CollectorData
*/
public function getCollectedData(): array
{
Expand Down
11 changes: 5 additions & 6 deletions src/Analyser/FileAnalyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@
use const E_USER_WARNING;
use const E_WARNING;

/**
* @phpstan-import-type CollectorData from CollectedData
*/
final class FileAnalyser
{

Expand Down Expand Up @@ -76,7 +79,7 @@ public function analyseFile(
/** @var list<Error> $locallyIgnoredErrors */
$locallyIgnoredErrors = [];

/** @var list<CollectedData> $fileCollectedData */
/** @var CollectorData $fileCollectedData */
$fileCollectedData = [];

$fileDependencies = [];
Expand Down Expand Up @@ -195,11 +198,7 @@ public function analyseFile(
continue;
}

$fileCollectedData[] = new CollectedData(
$collectedData,
$scope->getFile(),
get_class($collector),
);
$fileCollectedData[$scope->getFile()][get_class($collector)][] = $collectedData;
}

try {
Expand Down
5 changes: 3 additions & 2 deletions src/Analyser/FileAnalyserResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

/**
* @phpstan-type LinesToIgnore = array<string, array<int, non-empty-list<string>|null>>
* @phpstan-import-type CollectorData from CollectedData
*/
final class FileAnalyserResult
{
Expand All @@ -16,7 +17,7 @@ final class FileAnalyserResult
* @param list<Error> $filteredPhpErrors
* @param list<Error> $allPhpErrors
* @param list<Error> $locallyIgnoredErrors
* @param list<CollectedData> $collectedData
* @param CollectorData $collectedData
* @param list<string> $dependencies
* @param list<RootExportedNode> $exportedNodes
* @param LinesToIgnore $linesToIgnore
Expand Down Expand Up @@ -69,7 +70,7 @@ public function getLocallyIgnoredErrors(): array
}

/**
* @return list<CollectedData>
* @return CollectorData
*/
public function getCollectedData(): array
{
Expand Down
5 changes: 3 additions & 2 deletions src/Analyser/ResultCache/ResultCache.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

/**
* @phpstan-import-type LinesToIgnore from FileAnalyserResult
* @phpstan-import-type CollectorData from CollectedData
*/
final class ResultCache
{
Expand All @@ -20,7 +21,7 @@ final class ResultCache
* @param array<string, list<Error>> $locallyIgnoredErrors
* @param array<string, LinesToIgnore> $linesToIgnore
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
* @param array<string, array<CollectedData>> $collectedData
* @param CollectorData $collectedData
* @param array<string, array<string>> $dependencies
* @param array<string, array<RootExportedNode>> $exportedNodes
* @param array<string, array{string, bool, string}> $projectExtensionFiles
Expand Down Expand Up @@ -101,7 +102,7 @@ public function getUnmatchedLineIgnores(): array
}

/**
* @return array<string, array<CollectedData>>
* @return CollectorData
*/
public function getCollectedData(): array
{
Expand Down
25 changes: 10 additions & 15 deletions src/Analyser/ResultCache/ResultCacheManager.php
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@

/**
* @phpstan-import-type LinesToIgnore from FileAnalyserResult
* @phpstan-import-type CollectorData from CollectedData
*/
final class ResultCacheManager
{
Expand Down Expand Up @@ -406,10 +407,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
$freshLocallyIgnoredErrorsByFile[$error->getFilePath()][] = $error;
}

$freshCollectedDataByFile = [];
foreach ($analyserResult->getCollectedData() as $collectedData) {
$freshCollectedDataByFile[$collectedData->getFilePath()][] = $collectedData;
}
$freshCollectedDataByFile = $analyserResult->getCollectedData();

$meta = $resultCache->getMeta();
$projectConfigArray = $meta['projectConfig'];
Expand Down Expand Up @@ -524,13 +522,6 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
}
}

$flatCollectedData = [];
foreach ($collectedDataByFile as $fileCollectedData) {
foreach ($fileCollectedData as $collectedData) {
$flatCollectedData[] = $collectedData;
}
}

return new ResultCacheProcessResult(new AnalyserResult(
$flatErrors,
$analyserResult->getFilteredPhpErrors(),
Expand All @@ -539,7 +530,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
$linesToIgnore,
$unmatchedLineIgnores,
$internalErrors,
$flatCollectedData,
$collectedDataByFile,
$dependencies,
$exportedNodes,
$analyserResult->hasReachedInternalErrorsCountLimit(),
Expand Down Expand Up @@ -584,8 +575,8 @@ private function mergeLocallyIgnoredErrors(ResultCache $resultCache, array $fres
}

/**
* @param array<string, array<CollectedData>> $freshCollectedDataByFile
* @return array<string, array<CollectedData>>
* @param CollectorData $freshCollectedDataByFile
* @return CollectorData
*/
private function mergeCollectedData(ResultCache $resultCache, array $freshCollectedDataByFile): array
{
Expand Down Expand Up @@ -704,7 +695,7 @@ private function mergeUnmatchedLineIgnores(ResultCache $resultCache, array $fres
* @param array<string, list<Error>> $locallyIgnoredErrors
* @param array<string, LinesToIgnore> $linesToIgnore
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
* @param array<string, array<CollectedData>> $collectedData
* @param array<string, array<string, list<CollectedData>>> $collectedData
* @param array<string, array<string>> $dependencies
* @param array<string, array<RootExportedNode>> $exportedNodes
* @param array<string, array{string, bool, string}> $projectExtensionFiles
Expand Down Expand Up @@ -760,6 +751,10 @@ private function save(
ksort($collectedData);
ksort($invertedDependencies);

foreach ($collectedData as & $collectedDataPerFile) {
ksort($collectedDataPerFile);
}

foreach ($invertedDependencies as $file => $fileData) {
$dependentFiles = $fileData['dependentFiles'];
sort($dependentFiles);
Expand Down
2 changes: 2 additions & 0 deletions src/Collectors/CollectedData.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

/**
* @api
*
* @phpstan-type CollectorData = array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
*/
final class CollectedData implements JsonSerializable
{
Expand Down
22 changes: 21 additions & 1 deletion src/Command/AnalyseApplication.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use PHPStan\Analyser\AnalyserResultFinalizer;
use PHPStan\Analyser\Ignore\IgnoredErrorHelper;
use PHPStan\Analyser\ResultCache\ResultCacheManagerFactory;
use PHPStan\Collectors\CollectedData;
use PHPStan\Internal\BytesHelper;
use PHPStan\PhpDoc\StubFilesProvider;
use PHPStan\PhpDoc\StubValidator;
Expand All @@ -19,6 +20,9 @@
use function sha1_file;
use function sprintf;

/**
* @phpstan-import-type CollectorData from CollectedData
*/
final class AnalyseApplication
{

Expand Down Expand Up @@ -150,7 +154,7 @@ public function analyse(
$notFileSpecificErrors,
$internalErrors,
[],
$collectedData,
$this->mapCollectedData($collectedData),
$defaultLevelUsed,
$projectConfigFile,
$savedResultCache,
Expand All @@ -160,6 +164,22 @@ public function analyse(
);
}

/**
* @param CollectorData $collectedData
*
* @return list<CollectedData>
*/
private function mapCollectedData(array $collectedData): array
{
$result = [];
foreach ($collectedData as $file => $dataPerCollector) {
foreach ($dataPerCollector as $collectorType => $rawData) {
$result[] = new CollectedData($rawData, $file, $collectorType);
}
}
return $result;
}

/**
* @param string[] $files
* @param string[] $allAnalysedFiles
Expand Down
8 changes: 6 additions & 2 deletions src/Command/WorkerCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,12 @@ private function runWorker(
foreach ($fileAnalyserResult->getLocallyIgnoredErrors() as $locallyIgnoredError) {
$locallyIgnoredErrors[] = $locallyIgnoredError;
}
foreach ($fileAnalyserResult->getCollectedData() as $data) {
$collectedData[] = $data;
foreach ($fileAnalyserResult->getCollectedData() as $collectedFile => $dataPerCollector) {
foreach ($dataPerCollector as $collectorType => $collectorData) {
foreach ($collectorData as $data) {
$collectedData[$collectedFile][$collectorType][] = $data;
}
}
}
} catch (Throwable $t) {
$internalErrorsCount++;
Expand Down
15 changes: 6 additions & 9 deletions src/Node/CollectedDataNode.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,16 @@
use PhpParser\NodeAbstract;
use PHPStan\Collectors\CollectedData;
use PHPStan\Collectors\Collector;
use function array_key_exists;

/**
* @api
* @phpstan-import-type CollectorData from CollectedData
*/
final class CollectedDataNode extends NodeAbstract implements VirtualNode
{

/**
* @param CollectedData[] $collectedData
* @param CollectorData $collectedData
*/
public function __construct(private array $collectedData, private bool $onlyFiles)
{
Expand All @@ -31,17 +31,14 @@ public function __construct(private array $collectedData, private bool $onlyFile
public function get(string $collectorType): array
{
$result = [];
foreach ($this->collectedData as $collectedData) {
if ($collectedData->getCollectorType() !== $collectorType) {
foreach ($this->collectedData as $filePath => $collectedDataPerCollector) {
if (!isset($collectedDataPerCollector[$collectorType])) {
continue;
}

$filePath = $collectedData->getFilePath();
if (!array_key_exists($filePath, $result)) {
$result[$filePath] = [];
foreach ($collectedDataPerCollector[$collectorType] as $rawData) {
$result[$filePath][] = $rawData;
}

$result[$filePath][] = $collectedData->getData();
}

return $result;
Expand Down
9 changes: 6 additions & 3 deletions src/Parallel/ParallelAnalyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
use PHPStan\Analyser\AnalyserResult;
use PHPStan\Analyser\Error;
use PHPStan\Analyser\InternalError;
use PHPStan\Collectors\CollectedData;
use PHPStan\Dependency\RootExportedNode;
use PHPStan\Process\ProcessHelper;
use React\EventLoop\LoopInterface;
Expand Down Expand Up @@ -211,8 +210,12 @@ public function analyse(
$locallyIgnoredErrors[] = $locallyIgnoredFileError;
}

foreach ($json['collectedData'] as $jsonData) {
$collectedData[] = CollectedData::decode($jsonData);
foreach ($json['collectedData'] as $file => $jsonDataByCollector) {
foreach ($jsonDataByCollector as $collectorType => $listOfCollectedData) {
foreach ($listOfCollectedData as $rawCollectedData) {
$collectedData[$file][$collectorType][] = $rawCollectedData;
}
}
}

/**
Expand Down
Loading