Skip to content

Commit 97bc97f

Browse files
janedbaljanedbal
committed
Collected data: reduce memory consumption & result cache size
1 parent 6037f78 commit 97bc97f

File tree

10 files changed

+66
-49
lines changed

10 files changed

+66
-49
lines changed

‎src/Analyser/Analyser.php

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
namespace PHPStan\Analyser;
44

55
use Closure;
6-
use PHPStan\Collectors\CollectedData;
6+
use PhpParser\Node;
7+
use PHPStan\Collectors\Collector;
78
use PHPStan\Collectors\Registry as CollectorRegistry;
89
use PHPStan\Rules\Registry as RuleRegistry;
910
use Throwable;
@@ -59,7 +60,7 @@ public function analyse(
5960
$linesToIgnore = [];
6061
$unmatchedLineIgnores = [];
6162

62-
/** @var list<CollectedData> $collectedData */
63+
/** @var array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData */
6364
$collectedData = [];
6465

6566
$internalErrorsCount = 0;

‎src/Analyser/AnalyserResult.php

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
namespace PHPStan\Analyser;
44

5-
use PHPStan\Collectors\CollectedData;
5+
use PhpParser\Node;
6+
use PHPStan\Collectors\Collector;
67
use PHPStan\Dependency\RootExportedNode;
78
use function usort;
89

@@ -22,7 +23,7 @@ final class AnalyserResult
2223
* @param list<Error> $locallyIgnoredErrors
2324
* @param array<string, LinesToIgnore> $linesToIgnore
2425
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
25-
* @param list<CollectedData> $collectedData
26+
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
2627
* @param list<InternalError> $internalErrors
2728
* @param array<string, array<string>>|null $dependencies
2829
* @param array<string, array<RootExportedNode>> $exportedNodes
@@ -125,7 +126,7 @@ public function getInternalErrors(): array
125126
}
126127

127128
/**
128-
* @return list<CollectedData>
129+
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
129130
*/
130131
public function getCollectedData(): array
131132
{

‎src/Analyser/FileAnalyser.php

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
use PHPStan\BetterReflection\NodeCompiler\Exception\UnableToCompileNode;
88
use PHPStan\BetterReflection\Reflection\Exception\CircularReference;
99
use PHPStan\BetterReflection\Reflector\Exception\IdentifierNotFound;
10-
use PHPStan\Collectors\CollectedData;
10+
use PHPStan\Collectors\Collector;
1111
use PHPStan\Collectors\Registry as CollectorRegistry;
1212
use PHPStan\Dependency\DependencyResolver;
1313
use PHPStan\Node\FileNode;
@@ -76,7 +76,7 @@ public function analyseFile(
7676
/** @var list<Error> $locallyIgnoredErrors */
7777
$locallyIgnoredErrors = [];
7878

79-
/** @var list<CollectedData> $fileCollectedData */
79+
/** @var array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $fileCollectedData */
8080
$fileCollectedData = [];
8181

8282
$fileDependencies = [];
@@ -195,11 +195,7 @@ public function analyseFile(
195195
continue;
196196
}
197197

198-
$fileCollectedData[] = new CollectedData(
199-
$collectedData,
200-
$scope->getFile(),
201-
get_class($collector),
202-
);
198+
$fileCollectedData[$scope->getFile()][get_class($collector)][] = $collectedData;
203199
}
204200

205201
try {

‎src/Analyser/FileAnalyserResult.php

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
namespace PHPStan\Analyser;
44

5-
use PHPStan\Collectors\CollectedData;
5+
use PhpParser\Node;
6+
use PHPStan\Collectors\Collector;
67
use PHPStan\Dependency\RootExportedNode;
78

89
/**
@@ -16,7 +17,7 @@ final class FileAnalyserResult
1617
* @param list<Error> $filteredPhpErrors
1718
* @param list<Error> $allPhpErrors
1819
* @param list<Error> $locallyIgnoredErrors
19-
* @param list<CollectedData> $collectedData
20+
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
2021
* @param list<string> $dependencies
2122
* @param list<RootExportedNode> $exportedNodes
2223
* @param LinesToIgnore $linesToIgnore
@@ -69,7 +70,7 @@ public function getLocallyIgnoredErrors(): array
6970
}
7071

7172
/**
72-
* @return list<CollectedData>
73+
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
7374
*/
7475
public function getCollectedData(): array
7576
{

‎src/Analyser/ResultCache/ResultCache.php

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22

33
namespace PHPStan\Analyser\ResultCache;
44

5+
use PhpParser\Node;
56
use PHPStan\Analyser\Error;
67
use PHPStan\Analyser\FileAnalyserResult;
7-
use PHPStan\Collectors\CollectedData;
8+
use PHPStan\Collectors\Collector;
89
use PHPStan\Dependency\RootExportedNode;
910

1011
/**
@@ -20,7 +21,7 @@ final class ResultCache
2021
* @param array<string, list<Error>> $locallyIgnoredErrors
2122
* @param array<string, LinesToIgnore> $linesToIgnore
2223
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
23-
* @param array<string, array<CollectedData>> $collectedData
24+
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
2425
* @param array<string, array<string>> $dependencies
2526
* @param array<string, array<RootExportedNode>> $exportedNodes
2627
* @param array<string, array{string, bool, string}> $projectExtensionFiles
@@ -101,7 +102,7 @@ public function getUnmatchedLineIgnores(): array
101102
}
102103

103104
/**
104-
* @return array<string, array<CollectedData>>
105+
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
105106
*/
106107
public function getCollectedData(): array
107108
{

‎src/Analyser/ResultCache/ResultCacheManager.php

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
namespace PHPStan\Analyser\ResultCache;
44

55
use Nette\Neon\Neon;
6+
use PhpParser\Node;
67
use PHPStan\Analyser\AnalyserResult;
78
use PHPStan\Analyser\Error;
89
use PHPStan\Analyser\FileAnalyserResult;
910
use PHPStan\Collectors\CollectedData;
11+
use PHPStan\Collectors\Collector;
1012
use PHPStan\Command\Output;
1113
use PHPStan\Dependency\ExportedNodeFetcher;
1214
use PHPStan\Dependency\RootExportedNode;
@@ -406,10 +408,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
406408
$freshLocallyIgnoredErrorsByFile[$error->getFilePath()][] = $error;
407409
}
408410

409-
$freshCollectedDataByFile = [];
410-
foreach ($analyserResult->getCollectedData() as $collectedData) {
411-
$freshCollectedDataByFile[$collectedData->getFilePath()][] = $collectedData;
412-
}
411+
$freshCollectedDataByFile = $analyserResult->getCollectedData();
413412

414413
$meta = $resultCache->getMeta();
415414
$projectConfigArray = $meta['projectConfig'];
@@ -524,13 +523,6 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
524523
}
525524
}
526525

527-
$flatCollectedData = [];
528-
foreach ($collectedDataByFile as $fileCollectedData) {
529-
foreach ($fileCollectedData as $collectedData) {
530-
$flatCollectedData[] = $collectedData;
531-
}
532-
}
533-
534526
return new ResultCacheProcessResult(new AnalyserResult(
535527
$flatErrors,
536528
$analyserResult->getFilteredPhpErrors(),
@@ -539,7 +531,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
539531
$linesToIgnore,
540532
$unmatchedLineIgnores,
541533
$internalErrors,
542-
$flatCollectedData,
534+
$collectedDataByFile,
543535
$dependencies,
544536
$exportedNodes,
545537
$analyserResult->hasReachedInternalErrorsCountLimit(),
@@ -584,8 +576,8 @@ private function mergeLocallyIgnoredErrors(ResultCache $resultCache, array $fres
584576
}
585577

586578
/**
587-
* @param array<string, array<CollectedData>> $freshCollectedDataByFile
588-
* @return array<string, array<CollectedData>>
579+
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $freshCollectedDataByFile
580+
* @return array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
589581
*/
590582
private function mergeCollectedData(ResultCache $resultCache, array $freshCollectedDataByFile): array
591583
{
@@ -704,7 +696,7 @@ private function mergeUnmatchedLineIgnores(ResultCache $resultCache, array $fres
704696
* @param array<string, list<Error>> $locallyIgnoredErrors
705697
* @param array<string, LinesToIgnore> $linesToIgnore
706698
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
707-
* @param array<string, array<CollectedData>> $collectedData
699+
* @param array<string, array<string, list<CollectedData>>> $collectedData
708700
* @param array<string, array<string>> $dependencies
709701
* @param array<string, array<RootExportedNode>> $exportedNodes
710702
* @param array<string, array{string, bool, string}> $projectExtensionFiles
@@ -760,6 +752,10 @@ private function save(
760752
ksort($collectedData);
761753
ksort($invertedDependencies);
762754

755+
foreach ($collectedData as & $collectedDataPerFile) {
756+
ksort($collectedDataPerFile);
757+
}
758+
763759
foreach ($invertedDependencies as $file => $fileData) {
764760
$dependentFiles = $fileData['dependentFiles'];
765761
sort($dependentFiles);

‎src/Command/AnalyseApplication.php

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22

33
namespace PHPStan\Command;
44

5+
use PhpParser\Node;
56
use PHPStan\Analyser\AnalyserResult;
67
use PHPStan\Analyser\AnalyserResultFinalizer;
78
use PHPStan\Analyser\Ignore\IgnoredErrorHelper;
89
use PHPStan\Analyser\ResultCache\ResultCacheManagerFactory;
10+
use PHPStan\Collectors\CollectedData;
11+
use PHPStan\Collectors\Collector;
912
use PHPStan\Internal\BytesHelper;
1013
use PHPStan\PhpDoc\StubFilesProvider;
1114
use PHPStan\PhpDoc\StubValidator;
@@ -150,7 +153,7 @@ public function analyse(
150153
$notFileSpecificErrors,
151154
$internalErrors,
152155
[],
153-
$collectedData,
156+
$this->mapCollectedData($collectedData),
154157
$defaultLevelUsed,
155158
$projectConfigFile,
156159
$savedResultCache,
@@ -160,6 +163,22 @@ public function analyse(
160163
);
161164
}
162165

166+
/**
167+
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
168+
*
169+
* @return list<CollectedData>
170+
*/
171+
private function mapCollectedData(array $collectedData): array
172+
{
173+
$result = [];
174+
foreach ($collectedData as $file => $dataPerCollector) {
175+
foreach ($dataPerCollector as $collectorType => $rawData) {
176+
$result[] = new CollectedData($rawData, $file, $collectorType);
177+
}
178+
}
179+
return $result;
180+
}
181+
163182
/**
164183
* @param string[] $files
165184
* @param string[] $allAnalysedFiles

‎src/Command/WorkerCommand.php

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,12 @@ private function runWorker(
226226
foreach ($fileAnalyserResult->getLocallyIgnoredErrors() as $locallyIgnoredError) {
227227
$locallyIgnoredErrors[] = $locallyIgnoredError;
228228
}
229-
foreach ($fileAnalyserResult->getCollectedData() as $data) {
230-
$collectedData[] = $data;
229+
foreach ($fileAnalyserResult->getCollectedData() as $collectedFile => $dataPerCollector) {
230+
foreach ($dataPerCollector as $collectorType => $collectorData) {
231+
foreach ($collectorData as $data) {
232+
$collectedData[$collectedFile][$collectorType][] = $data;
233+
}
234+
}
231235
}
232236
} catch (Throwable $t) {
233237
$internalErrorsCount++;

‎src/Node/CollectedDataNode.php

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,7 @@
44

55
use PhpParser\Node;
66
use PhpParser\NodeAbstract;
7-
use PHPStan\Collectors\CollectedData;
87
use PHPStan\Collectors\Collector;
9-
use function array_key_exists;
108

119
/**
1210
* @api
@@ -15,7 +13,7 @@ final class CollectedDataNode extends NodeAbstract implements VirtualNode
1513
{
1614

1715
/**
18-
* @param CollectedData[] $collectedData
16+
* @param array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>> $collectedData
1917
*/
2018
public function __construct(private array $collectedData, private bool $onlyFiles)
2119
{
@@ -31,17 +29,14 @@ public function __construct(private array $collectedData, private bool $onlyFile
3129
public function get(string $collectorType): array
3230
{
3331
$result = [];
34-
foreach ($this->collectedData as $collectedData) {
35-
if ($collectedData->getCollectorType() !== $collectorType) {
32+
foreach ($this->collectedData as $filePath => $collectedDataPerCollector) {
33+
if (!isset($collectedDataPerCollector[$collectorType])) {
3634
continue;
3735
}
3836

39-
$filePath = $collectedData->getFilePath();
40-
if (!array_key_exists($filePath, $result)) {
41-
$result[$filePath] = [];
37+
foreach ($collectedDataPerCollector[$collectorType] as $rawData) {
38+
$result[$filePath][] = $rawData;
4239
}
43-
44-
$result[$filePath][] = $collectedData->getData();
4540
}
4641

4742
return $result;

‎src/Parallel/ParallelAnalyser.php

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
use PHPStan\Analyser\AnalyserResult;
1010
use PHPStan\Analyser\Error;
1111
use PHPStan\Analyser\InternalError;
12-
use PHPStan\Collectors\CollectedData;
1312
use PHPStan\Dependency\RootExportedNode;
1413
use PHPStan\Process\ProcessHelper;
1514
use React\EventLoop\LoopInterface;
@@ -211,8 +210,12 @@ public function analyse(
211210
$locallyIgnoredErrors[] = $locallyIgnoredFileError;
212211
}
213212

214-
foreach ($json['collectedData'] as $jsonData) {
215-
$collectedData[] = CollectedData::decode($jsonData);
213+
foreach ($json['collectedData'] as $file => $jsonDataByCollector) {
214+
foreach ($jsonDataByCollector as $collectorType => $listOfCollectedData) {
215+
foreach ($listOfCollectedData as $rawCollectedData) {
216+
$collectedData[$file][$collectorType][] = $rawCollectedData;
217+
}
218+
}
216219
}
217220

218221
/**

0 commit comments

Comments
 (0)