Skip to content

Commit f5631f6

Browse files
richard67richard67
committed
Fixes and improvements for deleted_file_check.php
- Add feature to use zip files without unpacking. - Let "to" parameter really be optional. - Create also lists of files and folders which have been added back. - Fix the help and doc blocks.
1 parent 84a6d3b commit f5631f6

File tree

1 file changed

+209
-78
lines changed

1 file changed

+209
-78
lines changed

‎build/deleted_file_check.php‎

Lines changed: 209 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,30 @@
11
<?php
22

33
/**
4-
* This file is used to build the list of deleted files between two reference points.
4+
* This file is used to build the lists of deleted files, deleted folders and
5+
* renamed files between two Joomla versions.
56
*
67
* This script requires one parameter:
78
*
8-
* --from - The git commit reference to use as the starting point for the comparison.
9+
* --from - Full package zip file or folder with unpacked full package of the
10+
* starting point for the comparison, i.e. the older version.
911
*
1012
* This script has one additional optional parameter:
1113
*
12-
* --to - The git commit reference to use as the ending point for the comparison.
14+
* --to - Full package zip file or folder with unpacked full package of the
15+
* ending point for the comparison, i.e. the newer version.
1316
*
14-
* The reference parameters may be any valid identifier (i.e. a branch, tag, or commit SHA)
17+
* If the "to" parameter is not given, the full package zip from a previous
18+
* run of the build script is used, if present.
1519
*
1620
* @package Joomla.Build
1721
*
1822
* @copyright (C) 2017 Open Source Matters, Inc. <https://www.joomla.org>
1923
* @license GNU General Public License version 2 or later; see LICENSE.txt
2024
*/
2125

26+
use Joomla\CMS\Version;
27+
2228
/*
2329
* Constants
2430
*/
@@ -28,8 +34,15 @@ function usage($command)
2834
{
2935
echo PHP_EOL;
3036
echo 'Usage: php ' . $command . ' [options]' . PHP_EOL;
31-
echo PHP_TAB . '--from <ref>:' . PHP_TAB . 'Starting commit reference (branch/tag)' . PHP_EOL;
32-
echo PHP_TAB . '--to <ref>:' . PHP_TAB . 'Ending commit reference (branch/tag) [optional]' . PHP_EOL;
37+
echo PHP_TAB . '--from <path>:' . PHP_TAB . 'Path to starting version' . PHP_EOL;
38+
echo PHP_TAB . '--to <path>:' . PHP_TAB . 'Path to ending version [optional]' . PHP_EOL;
39+
echo PHP_EOL;
40+
echo '<path> can be either of the following:' . PHP_EOL;
41+
echo PHP_TAB . '- Path to a full package Zip file.' . PHP_EOL;
42+
echo PHP_TAB . '- Path to a directory where a full package Zip file has been extracted to.' . PHP_EOL;
43+
echo PHP_EOL;
44+
echo 'If the "to" parameter is not specified, file "build/tmp/packages/*Full_Package.zip"' . PHP_EOL;
45+
echo 'is used if it exists from a previous run of the build script.' . PHP_EOL;
3346
echo PHP_EOL;
3447
}
3548

@@ -39,106 +52,179 @@ function usage($command)
3952

4053
$options = getopt('', ['from:', 'to::']);
4154

42-
// We need the from reference, otherwise we're doomed to fail
55+
// We need the "from" parameter, otherwise we're doomed to fail
4356
if (empty($options['from'])) {
4457
echo PHP_EOL;
45-
echo 'Missing starting directory' . PHP_EOL;
58+
echo 'Missing "from" parameter' . PHP_EOL;
4659

4760
usage($argv[0]);
4861

4962
exit(1);
5063
}
5164

52-
// Missing the to reference? No problem, grab the current HEAD
65+
// If the "to" parameter is not specified, use the default
5366
if (empty($options['to'])) {
67+
// Import the version class to get the version information
68+
\define('JPATH_PLATFORM', 1);
69+
require_once \dirname(__DIR__) . '/libraries/src/Version.php';
70+
71+
$fullVersion = (new Version())->getShortVersion();
72+
$packageStability = str_replace(' ', '_', Version::DEV_STATUS);
73+
$packageFile = __DIR__ . '/tmp/packages/Joomla_' . $fullVersion . '-' . $packageStability . '-Full_Package.zip';
74+
75+
if (is_file($packageFile)) {
76+
$options['to'] = $packageFile;
77+
} else {
78+
echo PHP_EOL;
79+
echo 'Missing "to" parameter and no zip file "' . $packageFile . '" found.' . PHP_EOL;
80+
81+
usage($argv[0]);
82+
83+
exit(1);
84+
}
85+
}
86+
87+
// Check from and to if folder or zip file
88+
if (!is_dir($options['from']) && !(is_file($options['from']) && substr(strtolower($options['from']), -4) === '.zip')) {
5489
echo PHP_EOL;
55-
echo 'Missing ending directory' . PHP_EOL;
90+
echo 'The "from" parameter is neither a directory nor a zip file' . PHP_EOL;
5691

57-
usage($argv[0]);
92+
exit(1);
93+
}
94+
95+
if (!is_dir($options['to']) && !(is_file($options['to']) && substr(strtolower($options['to']), -4) === '.zip')) {
96+
echo PHP_EOL;
97+
echo 'The "to" parameter is neither a directory nor a zip file' . PHP_EOL;
5898

5999
exit(1);
60100
}
61101

62-
// Directories to skip for the check (needs to include anything from previous versions which we want to keep)
63-
$previousReleaseExclude = [
64-
$options['from'] . '/images/sampledata',
65-
$options['from'] . '/installation',
66-
$options['from'] . '/media/plg_captcha_recaptcha',
67-
$options['from'] . '/media/plg_captcha_recaptcha_invisible',
102+
// Directories to skip for the check
103+
$excludedFolders = [
104+
'images/sampledata',
105+
'installation',
68106
];
69107

70108
/**
71-
* @param SplFileInfo $file The file being checked
72-
* @param mixed $key ?
73-
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
109+
* @param string $folderPath Path to the folder with the extracted full package
110+
* @param array $excludeFolders Excluded folders
74111
*
75-
* @return bool True if you need to recurse or if the item is acceptable
112+
* @return stdClass An object with arrays "files" and "folders"
76113
*/
77-
$previousReleaseFilter = function ($file, $key, $iterator) use ($previousReleaseExclude) {
78-
if ($iterator->hasChildren() && !\in_array($file->getPathname(), $previousReleaseExclude)) {
79-
return true;
114+
function readFolder($folderPath, $excludeFolders): stdClass
115+
{
116+
$return = new stdClass();
117+
118+
$return->files = [];
119+
$return->folders = [];
120+
121+
$skipFolders = [];
122+
123+
foreach ($excludeFolders as $excludeFolder) {
124+
$skipFolders[] = $folderPath . '/' . $excludeFolder;
80125
}
81126

82-
return $file->isFile();
83-
};
127+
/**
128+
* @param SplFileInfo $file The file being checked
129+
* @param mixed $key ?
130+
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
131+
*
132+
* @return bool True if you need to recurse or if the item is acceptable
133+
*/
134+
$releaseFilter = function ($file, $key, $iterator) use ($skipFolders) {
135+
if ($iterator->hasChildren() && !\in_array($file->getPathname(), $skipFolders)) {
136+
return true;
137+
}
84138

85-
// Directories to skip for the check
86-
$newReleaseExclude = [
87-
$options['to'] . '/installation',
88-
];
139+
return $file->isFile();
140+
};
141+
142+
$releaseDirIterator = new RecursiveDirectoryIterator($folderPath, RecursiveDirectoryIterator::SKIP_DOTS);
143+
$releaseIterator = new RecursiveIteratorIterator(
144+
new RecursiveCallbackFilterIterator($releaseDirIterator, $releaseFilter),
145+
RecursiveIteratorIterator::SELF_FIRST
146+
);
147+
148+
foreach ($releaseIterator as $info) {
149+
if ($info->isDir()) {
150+
$return->folders[] = "'" . str_replace($folderPath, '', $info->getPathname()) . "',";
151+
continue;
152+
}
153+
154+
$return->files[] = "'" . str_replace($folderPath, '', $info->getPathname()) . "',";
155+
}
156+
157+
return $return;
158+
}
89159

90160
/**
91-
* @param SplFileInfo $file The file being checked
92-
* @param mixed $key ?
93-
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
161+
* @param string $filePath Path to the full package zip file
162+
* @param array $excludeFolders Excluded folders
94163
*
95-
* @return bool True if you need to recurse or if the item is acceptable
164+
* @return stdClass An object with arrays "files" and "folders"
96165
*/
97-
$newReleaseFilter = function ($file, $key, $iterator) use ($newReleaseExclude) {
98-
if ($iterator->hasChildren() && !\in_array($file->getPathname(), $newReleaseExclude)) {
99-
return true;
100-
}
166+
function readZipFile($filePath, $excludeFolders): stdClass
167+
{
168+
$return = new stdClass();
101169

102-
return $file->isFile();
103-
};
170+
$return->files = [];
171+
$return->folders = [];
104172

105-
$previousReleaseDirIterator = new RecursiveDirectoryIterator($options['from'], RecursiveDirectoryIterator::SKIP_DOTS);
106-
$previousReleaseIterator = new RecursiveIteratorIterator(
107-
new RecursiveCallbackFilterIterator($previousReleaseDirIterator, $previousReleaseFilter),
108-
RecursiveIteratorIterator::SELF_FIRST
109-
);
110-
$previousReleaseFiles = [];
111-
$previousReleaseFolders = [];
173+
$zipArchive = new ZipArchive();
112174

113-
foreach ($previousReleaseIterator as $info) {
114-
if ($info->isDir()) {
115-
$previousReleaseFolders[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
116-
continue;
175+
if ($zipArchive->open($filePath) !== true) {
176+
echo PHP_EOL;
177+
echo 'Could not open zip archive "' . $filePath . '".' . PHP_EOL;
178+
179+
exit(1);
117180
}
118181

119-
$previousReleaseFiles[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
120-
}
182+
$excludeRegexp = '/^(';
121183

122-
$newReleaseDirIterator = new RecursiveDirectoryIterator($options['to'], RecursiveDirectoryIterator::SKIP_DOTS);
123-
$newReleaseIterator = new RecursiveIteratorIterator(
124-
new RecursiveCallbackFilterIterator($newReleaseDirIterator, $newReleaseFilter),
125-
RecursiveIteratorIterator::SELF_FIRST
126-
);
127-
$newReleaseFiles = [];
128-
$newReleaseFolders = [];
184+
foreach ($excludeFolders as $excludeFolder) {
185+
$excludeRegexp .= preg_quote($excludeFolder, '/') . '|';
186+
}
129187

130-
foreach ($newReleaseIterator as $info) {
131-
if ($info->isDir()) {
132-
$newReleaseFolders[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
133-
continue;
188+
$excludeRegexp = rtrim($excludeRegexp, '|') . ')\/.*/';
189+
190+
for ($i = 0; $i < $zipArchive->numFiles; $i++) {
191+
$stat = $zipArchive->statIndex($i);
192+
193+
$name = $stat['name'];
194+
195+
if (preg_match($excludeRegexp, $name) === 1) {
196+
continue;
197+
}
198+
199+
if (substr($name, -1) === '/') {
200+
$return->folders[] = "'/" . rtrim($name, '/') . "',";
201+
} else {
202+
$return->files[] = "'/" . $name . "',";
203+
}
134204
}
135205

136-
$newReleaseFiles[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
206+
$zipArchive->close();
207+
208+
return $return;
209+
}
210+
211+
// Read files and folders lists from folders or zip files
212+
if (is_dir($options['from'])) {
213+
$previousReleaseFilesFolders = readFolder($options['from'], $excludedFolders);
214+
} else {
215+
$previousReleaseFilesFolders = readZipFile($options['from'], $excludedFolders);
137216
}
138217

139-
$filesDifference = array_diff($previousReleaseFiles, $newReleaseFiles);
218+
if (is_dir($options['to'])) {
219+
$newReleaseFilesFolders = readFolder($options['to'], $excludedFolders);
220+
} else {
221+
$newReleaseFilesFolders = readZipFile($options['to'], $excludedFolders);
222+
}
140223

141-
$foldersDifference = array_diff($previousReleaseFolders, $newReleaseFolders);
224+
$filesDifferenceAdd = array_diff($newReleaseFilesFolders->files, $previousReleaseFilesFolders->files);
225+
$filesDifferenceDelete = array_diff($previousReleaseFilesFolders->files, $newReleaseFilesFolders->files);
226+
$foldersDifferenceAdd = array_diff($newReleaseFilesFolders->folders, $previousReleaseFilesFolders->folders);
227+
$foldersDifferenceDelete = array_diff($previousReleaseFilesFolders->folders, $newReleaseFilesFolders->folders);
142228

143229
// Specific files (e.g. language files) that we want to keep on upgrade
144230
$filesToKeep = [
@@ -152,33 +238,33 @@ function usage($command)
152238

153239
// Remove folders from the results which we want to keep on upgrade
154240
foreach ($foldersToKeep as $folder) {
155-
if (($key = array_search($folder, $foldersDifference)) !== false) {
156-
unset($foldersDifference[$key]);
241+
if (($key = array_search($folder, $foldersDifferenceDelete)) !== false) {
242+
unset($foldersDifferenceDelete[$key]);
157243
}
158244
}
159245

160-
asort($filesDifference);
161-
rsort($foldersDifference);
246+
asort($filesDifferenceDelete);
247+
rsort($foldersDifferenceDelete);
162248

163249
$deletedFiles = [];
164250
$renamedFiles = [];
165251

166-
foreach ($filesDifference as $file) {
252+
foreach ($filesDifferenceDelete as $file) {
167253
// Don't remove any specific files (e.g. language files) that we want to keep on upgrade
168254
if (array_search($file, $filesToKeep) !== false) {
169255
continue;
170256
}
171257

172258
// Check for files which might have been renamed only
173-
$matches = preg_grep('/^' . preg_quote($file, '/') . '$/i', $newReleaseFiles);
259+
$matches = preg_grep('/^' . preg_quote($file, '/') . '$/i', $newReleaseFilesFolders->files);
174260

175261
if ($matches !== false) {
176262
foreach ($matches as $match) {
177263
if (\dirname($match) === \dirname($file) && strtolower(basename($match)) === strtolower(basename($file))) {
178264
// File has been renamed only: Add to renamed files list
179265
$renamedFiles[] = substr($file, 0, -1) . ' => ' . $match;
180266

181-
// Go on with the next file in $filesDifference
267+
// Go on with the next file in $filesDifferenceDelete
182268
continue 2;
183269
}
184270
}
@@ -189,9 +275,54 @@ function usage($command)
189275
}
190276

191277
// Write the lists to files for later reference
192-
file_put_contents(__DIR__ . '/deleted_files.txt', implode("\n", $deletedFiles));
193-
file_put_contents(__DIR__ . '/deleted_folders.txt', implode("\n", $foldersDifference));
194-
file_put_contents(__DIR__ . '/renamed_files.txt', implode("\n", $renamedFiles));
278+
$addedFilesFile = __DIR__ . '/added_files.txt';
279+
$addedFoldersFile = __DIR__ . '/added_folders.txt';
280+
$deletedFilesFile = __DIR__ . '/deleted_files.txt';
281+
$deletedFoldersFile = __DIR__ . '/deleted_folders.txt';
282+
$renamedFilesFile = __DIR__ . '/renamed_files.txt';
283+
284+
@unlink($addedFilesFile);
285+
@unlink($addedFoldersFile);
286+
@unlink($deletedFilesFile);
287+
@unlink($deletedFoldersFile);
288+
@unlink($renamedFilesFile);
289+
290+
if (\count($filesDifferenceAdd) > 0) {
291+
file_put_contents($addedFilesFile, implode("\n", $filesDifferenceAdd));
292+
}
293+
294+
if (\count($foldersDifferenceAdd) > 0) {
295+
file_put_contents($addedFoldersFile, implode("\n", $foldersDifferenceAdd));
296+
}
297+
298+
if (\count($deletedFiles) > 0) {
299+
file_put_contents($deletedFilesFile, implode("\n", $deletedFiles));
300+
}
301+
302+
if (\count($foldersDifferenceDelete) > 0) {
303+
file_put_contents($deletedFoldersFile, implode("\n", $foldersDifferenceDelete));
304+
}
305+
306+
if (\count($renamedFiles) > 0) {
307+
file_put_contents($renamedFilesFile, implode("\n", $renamedFiles));
308+
}
309+
310+
echo PHP_EOL;
311+
echo 'There are ' . PHP_EOL;
312+
echo ' - ' . \count($filesDifferenceAdd) . ' added files, ' . PHP_EOL;
313+
echo ' - ' . \count($foldersDifferenceAdd) . ' added folders, ' . PHP_EOL;
314+
echo ' - ' . \count($deletedFiles) . ' deleted files, ' . PHP_EOL;
315+
echo ' - ' . \count($foldersDifferenceDelete) . ' deleted folders and ' . PHP_EOL;
316+
echo ' - ' . \count($renamedFiles) . ' renamed files' . PHP_EOL;
317+
echo PHP_EOL;
318+
echo 'in comparison' . PHP_EOL;
319+
echo ' from "' . $options['from'] . '"' . PHP_EOL;
320+
echo ' to "' . $options['to'] . '"' . PHP_EOL;
321+
echo PHP_EOL;
322+
echo 'The following folders and their subfolders have been skipped so they were not included in the comparison:' . PHP_EOL;
323+
324+
foreach ($excludedFolders as $excludedFolder) {
325+
echo ' - ' . $excludedFolder . PHP_EOL;
326+
}
195327

196328
echo PHP_EOL;
197-
echo 'There are ' . \count($deletedFiles) . ' deleted files, ' . \count($foldersDifference) . ' deleted folders and ' . \count($renamedFiles) . ' renamed files in comparison to "' . $options['from'] . '"' . PHP_EOL;

0 commit comments

Comments
 (0)