Skip to content

Commit c8bc331

Browse files
richard67richard67
committed
Fixes and improvements for deleted_file_check.php
- Add feature to use zip files without unpacking. - Let "to" parameter really be optional. - Create also lists of files and folders which have been added back. - Fix the help and doc blocks.
1 parent b7d5d5a commit c8bc331

File tree

1 file changed

+213
-80
lines changed

1 file changed

+213
-80
lines changed

‎build/deleted_file_check.php‎

Lines changed: 213 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,30 @@
11
<?php
22

33
/**
4-
* This file is used to build the list of deleted files between two reference points.
4+
* This file is used to build the lists of deleted files, deleted folders and
5+
* renamed files between two Joomla versions.
56
*
67
* This script requires one parameter:
78
*
8-
* --from - The git commit reference to use as the starting point for the comparison.
9+
* --from - Full package zip file or folder with unpacked full package of the
10+
* starting point for the comparison, i.e. the older version.
911
*
1012
* This script has one additional optional parameter:
1113
*
12-
* --to - The git commit reference to use as the ending point for the comparison.
14+
* --to - Full package zip file or folder with unpacked full package of the
15+
* ending point for the comparison, i.e. the newer version.
1316
*
14-
* The reference parameters may be any valid identifier (i.e. a branch, tag, or commit SHA)
17+
* If the "to" parameter is not given, the full package zip from a previous
18+
* run of the build script is used, if present.
1519
*
1620
* @package Joomla.Build
1721
*
1822
* @copyright (C) 2017 Open Source Matters, Inc. <https://www.joomla.org>
1923
* @license GNU General Public License version 2 or later; see LICENSE.txt
2024
*/
2125

26+
use Joomla\CMS\Version;
27+
2228
/*
2329
* Constants
2430
*/
@@ -28,8 +34,15 @@ function usage($command)
2834
{
2935
echo PHP_EOL;
3036
echo 'Usage: php ' . $command . ' [options]' . PHP_EOL;
31-
echo PHP_TAB . '--from <ref>:' . PHP_TAB . 'Starting commit reference (branch/tag)' . PHP_EOL;
32-
echo PHP_TAB . '--to <ref>:' . PHP_TAB . 'Ending commit reference (branch/tag) [optional]' . PHP_EOL;
37+
echo PHP_TAB . '--from <path>:' . PHP_TAB . 'Path to starting version' . PHP_EOL;
38+
echo PHP_TAB . '--to <path>:' . PHP_TAB . 'Path to ending version [optional]' . PHP_EOL;
39+
echo PHP_EOL;
40+
echo '<path> can be either of the following:' . PHP_EOL;
41+
echo PHP_TAB . '- Path to a full package Zip file.' . PHP_EOL;
42+
echo PHP_TAB . '- Path to a directory where a full package Zip file has been extracted to.' . PHP_EOL;
43+
echo PHP_EOL;
44+
echo 'If the "to" parameter is not specified, file "build/tmp/packages/*Full_Package.zip"' . PHP_EOL;
45+
echo 'is used if it exists from a previous run of the build script.' . PHP_EOL;
3346
echo PHP_EOL;
3447
}
3548

@@ -39,108 +52,183 @@ function usage($command)
3952

4053
$options = getopt('', ['from:', 'to::']);
4154

42-
// We need the from reference, otherwise we're doomed to fail
55+
// We need the "from" parameter, otherwise we're doomed to fail
4356
if (empty($options['from'])) {
4457
echo PHP_EOL;
45-
echo 'Missing starting directory' . PHP_EOL;
58+
echo 'Missing "from" parameter' . PHP_EOL;
4659

4760
usage($argv[0]);
4861

4962
exit(1);
5063
}
5164

52-
// Missing the to reference? No problem, grab the current HEAD
65+
// If the "to" parameter is not specified, use the default
5366
if (empty($options['to'])) {
67+
// Import the version class to get the version information
68+
\define('JPATH_PLATFORM', 1);
69+
require_once \dirname(__DIR__) . '/libraries/src/Version.php';
70+
71+
$fullVersion = (new Version())->getShortVersion();
72+
$packageStability = str_replace(' ', '_', Version::DEV_STATUS);
73+
$packageFile = __DIR__ . '/tmp/packages/Joomla_' . $fullVersion . '-' . $packageStability . '-Full_Package.zip';
74+
75+
if (is_file($packageFile)) {
76+
$options['to'] = $packageFile;
77+
} else {
78+
echo PHP_EOL;
79+
echo 'Missing "to" parameter and no zip file "' . $packageFile . '" found.' . PHP_EOL;
80+
81+
usage($argv[0]);
82+
83+
exit(1);
84+
}
85+
}
86+
87+
// Check from and to if folder or zip file
88+
if (!is_dir($options['from']) && !(is_file($options['from']) && substr(strtolower($options['from']), -4) === '.zip')) {
5489
echo PHP_EOL;
55-
echo 'Missing ending directory' . PHP_EOL;
90+
echo 'The "from" parameter is neither a directory nor a zip file' . PHP_EOL;
5691

57-
usage($argv[0]);
92+
exit(1);
93+
}
94+
95+
if (!is_dir($options['to']) && !(is_file($options['to']) && substr(strtolower($options['to']), -4) === '.zip')) {
96+
echo PHP_EOL;
97+
echo 'The "to" parameter is neither a directory nor a zip file' . PHP_EOL;
5898

5999
exit(1);
60100
}
61101

62-
// Directories to skip for the check (needs to include anything from previous versions which we want to keep)
63-
$previousReleaseExclude = [
64-
$options['from'] . '/images/sampledata',
65-
$options['from'] . '/installation',
66-
$options['from'] . '/media/plg_captcha_recaptcha',
67-
$options['from'] . '/media/plg_captcha_recaptcha_invisible',
68-
$options['from'] . '/media/plg_behaviour_compat',
69-
$options['from'] . '/plugins/behaviour/compat',
102+
// Directories to skip for the check
103+
$excludedFolders = [
104+
'images/sampledata',
105+
'installation',
106+
'/media/plg_captcha_recaptcha',
107+
'/media/plg_captcha_recaptcha_invisible',
108+
'/media/plg_behaviour_compat',
109+
'/plugins/behaviour/compat',
70110
];
71111

72112
/**
73-
* @param SplFileInfo $file The file being checked
74-
* @param mixed $key ?
75-
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
113+
* @param string $folderPath Path to the folder with the extracted full package
114+
* @param array $excludeFolders Excluded folders
76115
*
77-
* @return bool True if you need to recurse or if the item is acceptable
116+
* @return stdClass An object with arrays "files" and "folders"
78117
*/
79-
$previousReleaseFilter = function ($file, $key, $iterator) use ($previousReleaseExclude) {
80-
if ($iterator->hasChildren() && !\in_array($file->getPathname(), $previousReleaseExclude)) {
81-
return true;
118+
function readFolder($folderPath, $excludeFolders): stdClass
119+
{
120+
$return = new stdClass();
121+
122+
$return->files = [];
123+
$return->folders = [];
124+
125+
$skipFolders = [];
126+
127+
foreach ($excludeFolders as $excludeFolder) {
128+
$skipFolders[] = $folderPath . '/' . $excludeFolder;
82129
}
83130

84-
return $file->isFile();
85-
};
131+
/**
132+
* @param SplFileInfo $file The file being checked
133+
* @param mixed $key ?
134+
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
135+
*
136+
* @return bool True if you need to recurse or if the item is acceptable
137+
*/
138+
$releaseFilter = function ($file, $key, $iterator) use ($skipFolders) {
139+
if ($iterator->hasChildren() && !\in_array($file->getPathname(), $skipFolders)) {
140+
return true;
141+
}
86142

87-
// Directories to skip for the check
88-
$newReleaseExclude = [
89-
$options['to'] . '/installation',
90-
];
143+
return $file->isFile();
144+
};
145+
146+
$releaseDirIterator = new RecursiveDirectoryIterator($folderPath, RecursiveDirectoryIterator::SKIP_DOTS);
147+
$releaseIterator = new RecursiveIteratorIterator(
148+
new RecursiveCallbackFilterIterator($releaseDirIterator, $releaseFilter),
149+
RecursiveIteratorIterator::SELF_FIRST
150+
);
151+
152+
foreach ($releaseIterator as $info) {
153+
if ($info->isDir()) {
154+
$return->folders[] = "'" . str_replace($folderPath, '', $info->getPathname()) . "',";
155+
continue;
156+
}
157+
158+
$return->files[] = "'" . str_replace($folderPath, '', $info->getPathname()) . "',";
159+
}
160+
161+
return $return;
162+
}
91163

92164
/**
93-
* @param SplFileInfo $file The file being checked
94-
* @param mixed $key ?
95-
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
165+
* @param string $filePath Path to the full package zip file
166+
* @param array $excludeFolders Excluded folders
96167
*
97-
* @return bool True if you need to recurse or if the item is acceptable
168+
* @return stdClass An object with arrays "files" and "folders"
98169
*/
99-
$newReleaseFilter = function ($file, $key, $iterator) use ($newReleaseExclude) {
100-
if ($iterator->hasChildren() && !\in_array($file->getPathname(), $newReleaseExclude)) {
101-
return true;
102-
}
170+
function readZipFile($filePath, $excludeFolders): stdClass
171+
{
172+
$return = new stdClass();
103173

104-
return $file->isFile();
105-
};
174+
$return->files = [];
175+
$return->folders = [];
106176

107-
$previousReleaseDirIterator = new RecursiveDirectoryIterator($options['from'], RecursiveDirectoryIterator::SKIP_DOTS);
108-
$previousReleaseIterator = new RecursiveIteratorIterator(
109-
new RecursiveCallbackFilterIterator($previousReleaseDirIterator, $previousReleaseFilter),
110-
RecursiveIteratorIterator::SELF_FIRST
111-
);
112-
$previousReleaseFiles = [];
113-
$previousReleaseFolders = [];
177+
$zipArchive = new ZipArchive();
114178

115-
foreach ($previousReleaseIterator as $info) {
116-
if ($info->isDir()) {
117-
$previousReleaseFolders[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
118-
continue;
179+
if ($zipArchive->open($filePath) !== true) {
180+
echo PHP_EOL;
181+
echo 'Could not open zip archive "' . $filePath . '".' . PHP_EOL;
182+
183+
exit(1);
119184
}
120185

121-
$previousReleaseFiles[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
122-
}
186+
$excludeRegexp = '/^(';
123187

124-
$newReleaseDirIterator = new RecursiveDirectoryIterator($options['to'], RecursiveDirectoryIterator::SKIP_DOTS);
125-
$newReleaseIterator = new RecursiveIteratorIterator(
126-
new RecursiveCallbackFilterIterator($newReleaseDirIterator, $newReleaseFilter),
127-
RecursiveIteratorIterator::SELF_FIRST
128-
);
129-
$newReleaseFiles = [];
130-
$newReleaseFolders = [];
188+
foreach ($excludeFolders as $excludeFolder) {
189+
$excludeRegexp .= preg_quote($excludeFolder, '/') . '|';
190+
}
131191

132-
foreach ($newReleaseIterator as $info) {
133-
if ($info->isDir()) {
134-
$newReleaseFolders[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
135-
continue;
192+
$excludeRegexp = rtrim($excludeRegexp, '|') . ')\/.*/';
193+
194+
for ($i = 0; $i < $zipArchive->numFiles; $i++) {
195+
$stat = $zipArchive->statIndex($i);
196+
197+
$name = $stat['name'];
198+
199+
if (preg_match($excludeRegexp, $name) === 1) {
200+
continue;
201+
}
202+
203+
if (substr($name, -1) === '/') {
204+
$return->folders[] = "'/" . rtrim($name, '/') . "',";
205+
} else {
206+
$return->files[] = "'/" . $name . "',";
207+
}
136208
}
137209

138-
$newReleaseFiles[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
210+
$zipArchive->close();
211+
212+
return $return;
213+
}
214+
215+
// Read files and folders lists from folders or zip files
216+
if (is_dir($options['from'])) {
217+
$previousReleaseFilesFolders = readFolder($options['from'], $excludedFolders);
218+
} else {
219+
$previousReleaseFilesFolders = readZipFile($options['from'], $excludedFolders);
139220
}
140221

141-
$filesDifference = array_diff($previousReleaseFiles, $newReleaseFiles);
222+
if (is_dir($options['to'])) {
223+
$newReleaseFilesFolders = readFolder($options['to'], $excludedFolders);
224+
} else {
225+
$newReleaseFilesFolders = readZipFile($options['to'], $excludedFolders);
226+
}
142227

143-
$foldersDifference = array_diff($previousReleaseFolders, $newReleaseFolders);
228+
$filesDifferenceAdd = array_diff($newReleaseFilesFolders->files, $previousReleaseFilesFolders->files);
229+
$filesDifferenceDelete = array_diff($previousReleaseFilesFolders->files, $newReleaseFilesFolders->files);
230+
$foldersDifferenceAdd = array_diff($newReleaseFilesFolders->folders, $previousReleaseFilesFolders->folders);
231+
$foldersDifferenceDelete = array_diff($previousReleaseFilesFolders->folders, $newReleaseFilesFolders->folders);
144232

145233
// Specific files (e.g. language files) that we want to keep on upgrade
146234
$filesToKeep = [
@@ -156,33 +244,33 @@ function usage($command)
156244

157245
// Remove folders from the results which we want to keep on upgrade
158246
foreach ($foldersToKeep as $folder) {
159-
if (($key = array_search($folder, $foldersDifference)) !== false) {
160-
unset($foldersDifference[$key]);
247+
if (($key = array_search($folder, $foldersDifferenceDelete)) !== false) {
248+
unset($foldersDifferenceDelete[$key]);
161249
}
162250
}
163251

164-
asort($filesDifference);
165-
rsort($foldersDifference);
252+
asort($filesDifferenceDelete);
253+
rsort($foldersDifferenceDelete);
166254

167255
$deletedFiles = [];
168256
$renamedFiles = [];
169257

170-
foreach ($filesDifference as $file) {
258+
foreach ($filesDifferenceDelete as $file) {
171259
// Don't remove any specific files (e.g. language files) that we want to keep on upgrade
172260
if (array_search($file, $filesToKeep) !== false) {
173261
continue;
174262
}
175263

176264
// Check for files which might have been renamed only
177-
$matches = preg_grep('/^' . preg_quote($file, '/') . '$/i', $newReleaseFiles);
265+
$matches = preg_grep('/^' . preg_quote($file, '/') . '$/i', $newReleaseFilesFolders->files);
178266

179267
if ($matches !== false) {
180268
foreach ($matches as $match) {
181269
if (\dirname($match) === \dirname($file) && strtolower(basename($match)) === strtolower(basename($file))) {
182270
// File has been renamed only: Add to renamed files list
183271
$renamedFiles[] = substr($file, 0, -1) . ' => ' . $match;
184272

185-
// Go on with the next file in $filesDifference
273+
// Go on with the next file in $filesDifferenceDelete
186274
continue 2;
187275
}
188276
}
@@ -193,9 +281,54 @@ function usage($command)
193281
}
194282

195283
// Write the lists to files for later reference
196-
file_put_contents(__DIR__ . '/deleted_files.txt', implode("\n", $deletedFiles));
197-
file_put_contents(__DIR__ . '/deleted_folders.txt', implode("\n", $foldersDifference));
198-
file_put_contents(__DIR__ . '/renamed_files.txt', implode("\n", $renamedFiles));
284+
$addedFilesFile = __DIR__ . '/added_files.txt';
285+
$addedFoldersFile = __DIR__ . '/added_folders.txt';
286+
$deletedFilesFile = __DIR__ . '/deleted_files.txt';
287+
$deletedFoldersFile = __DIR__ . '/deleted_folders.txt';
288+
$renamedFilesFile = __DIR__ . '/renamed_files.txt';
289+
290+
@unlink($addedFilesFile);
291+
@unlink($addedFoldersFile);
292+
@unlink($deletedFilesFile);
293+
@unlink($deletedFoldersFile);
294+
@unlink($renamedFilesFile);
295+
296+
if (\count($filesDifferenceAdd) > 0) {
297+
file_put_contents($addedFilesFile, implode("\n", $filesDifferenceAdd));
298+
}
299+
300+
if (\count($foldersDifferenceAdd) > 0) {
301+
file_put_contents($addedFoldersFile, implode("\n", $foldersDifferenceAdd));
302+
}
303+
304+
if (\count($deletedFiles) > 0) {
305+
file_put_contents($deletedFilesFile, implode("\n", $deletedFiles));
306+
}
307+
308+
if (\count($foldersDifferenceDelete) > 0) {
309+
file_put_contents($deletedFoldersFile, implode("\n", $foldersDifferenceDelete));
310+
}
311+
312+
if (\count($renamedFiles) > 0) {
313+
file_put_contents($renamedFilesFile, implode("\n", $renamedFiles));
314+
}
315+
316+
echo PHP_EOL;
317+
echo 'There are ' . PHP_EOL;
318+
echo ' - ' . \count($filesDifferenceAdd) . ' added files, ' . PHP_EOL;
319+
echo ' - ' . \count($foldersDifferenceAdd) . ' added folders, ' . PHP_EOL;
320+
echo ' - ' . \count($deletedFiles) . ' deleted files, ' . PHP_EOL;
321+
echo ' - ' . \count($foldersDifferenceDelete) . ' deleted folders and ' . PHP_EOL;
322+
echo ' - ' . \count($renamedFiles) . ' renamed files' . PHP_EOL;
323+
echo PHP_EOL;
324+
echo 'in comparison' . PHP_EOL;
325+
echo ' from "' . $options['from'] . '"' . PHP_EOL;
326+
echo ' to "' . $options['to'] . '"' . PHP_EOL;
327+
echo PHP_EOL;
328+
echo 'The following folders and their subfolders have been skipped so they were not included in the comparison:' . PHP_EOL;
329+
330+
foreach ($excludedFolders as $excludedFolder) {
331+
echo ' - ' . $excludedFolder . PHP_EOL;
332+
}
199333

200334
echo PHP_EOL;
201-
echo 'There are ' . \count($deletedFiles) . ' deleted files, ' . \count($foldersDifference) . ' deleted folders and ' . \count($renamedFiles) . ' renamed files in comparison to "' . $options['from'] . '"' . PHP_EOL;

0 commit comments

Comments
 (0)