Skip to content
This repository was archived by the owner on Nov 25, 2020. It is now read-only.

Commit 6df570b

Browse files
snw35snw35
committed
Commit to add tar binary support for compression and extraction of tar archive files. This allows the compression plugin to use the tar binary (if found) for these operations, which improves performance. It also supports and detects the parallel compression utilities: pigz, lbzip2, and pbzip2. If any of these are installed, then they will be used to compress or extract archive files while utilizing all available CPU threads on the system, increasing performance even further. This will work natively in nearly all flavours of Linux, and Windows is also supported as long as a usable tar binary is present in the command path.
1 parent c86f299 commit 6df570b

File tree

1 file changed

+146
-50
lines changed

1 file changed

+146
-50
lines changed

‎core/src/plugins/action.compression/class.PluginCompression.php

Lines changed: 146 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,32 @@ public function receiveAction($action, $httpVars, $fileVars)
4444
$currentDirPath = AJXP_Utils::safeDirname($userSelection->getUniqueNode()->getPath());
4545
$currentDirPath = rtrim($currentDirPath, "/") . "/";
4646
$currentDirUrl = $userSelection->currentBaseUrl().$currentDirPath;
47+
48+
# Multi-platform function taken from stackoverflow
49+
# http://stackoverflow.com/questions/12424787/how-to-check-if-a-shell-command-exists-from-php
50+
# Check that a command exists on the system for use with shell_exec
51+
function command_exists ($command) {
52+
$whereIsCommand = (PHP_OS == 'WINNT') ? 'where' : 'which';
53+
$process = proc_open(
54+
"$whereIsCommand $command",
55+
array(
56+
0 => array("pipe", "r"), //STDIN
57+
1 => array("pipe", "w"), //STDOUT
58+
2 => array("pipe", "w"), //STDERR
59+
),
60+
$pipes
61+
);
62+
if ($process !== false) {
63+
$stdout = stream_get_contents($pipes[1]);
64+
$stderr = stream_get_contents($pipes[2]);
65+
fclose($pipes[1]);
66+
fclose($pipes[2]);
67+
proc_close($process);
68+
return $stdout != '';
69+
}
70+
return false;
71+
}
72+
4773
if (empty($httpVars["compression_id"])) {
4874
$compressionId = sha1(rand());
4975
$httpVars["compression_id"] = $compressionId;
@@ -119,44 +145,85 @@ public function receiveAction($action, $httpVars, $fileVars)
119145
file_put_contents($progressCompressionFileName, "Error : " . $messages["compression.17"]);
120146
throw new AJXP_Exception($messages["compression.17"]);
121147
}
122-
try {
123-
$tmpArchiveName = tempnam(AJXP_Utils::getAjxpTmpDir(), "tar-compression") . ".tar";
124-
$archive = new PharData($tmpArchiveName);
125-
} catch (Exception $e) {
126-
file_put_contents($progressCompressionFileName, "Error : " . $e->getMessage());
127-
throw $e;
128-
}
129-
$counterCompression = 0;
130-
//THE TWO ARRAY ARE MERGED FOR THE FOREACH LOOP
131-
$tabAllFiles = array_combine($tabAllRecursiveFiles, $tabFilesNames);
132-
foreach ($tabAllFiles as $fullPath => $fileName) {
148+
// Builds a shell command that changes to the workspace location in the filesystem, and then
149+
// compresses with tar and any parallel compression utilities that are installed.
150+
if (command_exists("tar")) {
151+
$FileNames = array();
152+
$FolderNames = array();
153+
foreach ($nodes as $node) {
154+
$nodeUrl = $node->getUrl();
155+
if (is_file($nodeUrl) && filesize($nodeUrl) < $maxAuthorizedSize) {
156+
array_push($FileNames, escapeshellarg(substr($nodeUrl, $currentDirUrlLength)));
157+
}
158+
if (is_dir($nodeUrl)) {
159+
array_push($FolderNames, escapeshellarg(substr($nodeUrl, $currentDirUrlLength)));
160+
}
161+
}
162+
if ($archiveFormat == ".tar.gz") {
163+
if (command_exists("pigz")) {
164+
$tarcommand = "tar -I pigz -cf ";
165+
} else {
166+
$tarcommand = "tar -czf ";
167+
}
168+
} elseif ($archiveFormat == ".tar.bz2") {
169+
if (command_exists("lbzip2")) {
170+
$tarcommand = "tar -I lbzip2 -cf ";
171+
} elseif (command_exists("pbzip2")) {
172+
$tarcommand = "tar -I pbzip2 -cf ";
173+
} else {
174+
$tarcommand = "tar -cjf ";
175+
}
176+
} elseif ($archiveFormat == ".tar") {
177+
$tarcommand = "tar -cf ";
178+
} else {
179+
file_put_contents($progressExtractFileName, "Error : " . $messages["compression.15"]);
180+
throw new AJXP_Exception($messages["compression.15"]);
181+
}
182+
$changedircommand = "cd " . AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl) . " && ";
183+
$finalcommand = $changedircommand . $tarcommand . escapeshellarg($archiveName) . " " . implode(" ", $FolderNames) . " " . implode(" ", $FileNames);
184+
shell_exec($finalcommand);
185+
} else {
186+
// If tar command is not found, compress with PHP Phar instead
133187
try {
134-
$archive->addFile(AJXP_MetaStreamWrapper::getRealFSReference($fullPath), $fileName);
135-
$counterCompression++;
136-
file_put_contents($progressCompressionFileName, sprintf($messages["compression.6"], round(($counterCompression / count($tabAllFiles)) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
188+
$tmpArchiveName = tempnam(AJXP_Utils::getAjxpTmpDir(), "tar-compression") . ".tar";
189+
$archive = new PharData($tmpArchiveName);
137190
} catch (Exception $e) {
138-
unlink($tmpArchiveName);
139191
file_put_contents($progressCompressionFileName, "Error : " . $e->getMessage());
140192
throw $e;
141193
}
142-
}
143-
$finalArchive = $tmpArchiveName;
144-
if ($typeArchive != ".tar") {
145-
$archiveTypeCompress = substr(strrchr($typeArchive, "."), 1);
146-
file_put_contents($progressCompressionFileName, sprintf($messages["compression.7"], strtoupper($archiveTypeCompress)));
147-
if ($archiveTypeCompress == "gz") {
148-
$archive->compress(Phar::GZ);
149-
} elseif ($archiveTypeCompress == "bz2") {
150-
$archive->compress(Phar::BZ2);
194+
$counterCompression = 0;
195+
//THE TWO ARRAY ARE MERGED FOR THE FOREACH LOOP
196+
$tabAllFiles = array_combine($tabAllRecursiveFiles, $tabFilesNames);
197+
foreach ($tabAllFiles as $fullPath => $fileName) {
198+
try {
199+
$archive->addFile(AJXP_MetaStreamWrapper::getRealFSReference($fullPath), $fileName);
200+
$counterCompression++;
201+
file_put_contents($progressCompressionFileName, sprintf($messages["compression.6"], round(($counterCompression / count($tabAllFiles)) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
202+
} catch (Exception $e) {
203+
unlink($tmpArchiveName);
204+
file_put_contents($progressCompressionFileName, "Error : " . $e->getMessage());
205+
throw $e;
206+
}
151207
}
152-
$finalArchive = $tmpArchiveName . "." . $archiveTypeCompress;
153-
}
154-
$destArchive = AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $archiveName);
155-
rename($finalArchive, $destArchive);
156-
AJXP_Controller::applyHook("node.before_create", array($destArchive, filesize($destArchive)));
157-
if (file_exists($tmpArchiveName)) {
158-
unlink($tmpArchiveName);
159-
unlink(substr($tmpArchiveName, 0, -4));
208+
$finalArchive = $tmpArchiveName;
209+
if ($typeArchive != ".tar") {
210+
$archiveTypeCompress = substr(strrchr($typeArchive, "."), 1);
211+
file_put_contents($progressCompressionFileName, sprintf($messages["compression.7"], strtoupper($archiveTypeCompress)));
212+
if ($archiveTypeCompress == "gz") {
213+
$archive->compress(Phar::GZ);
214+
} elseif ($archiveTypeCompress == "bz2") {
215+
$archive->compress(Phar::BZ2);
216+
}
217+
$finalArchive = $tmpArchiveName . "." . $archiveTypeCompress;
218+
}
219+
$destArchive = AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $archiveName);
220+
rename($finalArchive, $destArchive);
221+
AJXP_Controller::applyHook("node.before_create", array($destArchive, filesize($destArchive)));
222+
if (file_exists($tmpArchiveName)) {
223+
unlink($tmpArchiveName);
224+
unlink(substr($tmpArchiveName, 0, -4));
225+
}
226+
// End of Phar compression section
160227
}
161228
$newNode = new AJXP_Node($currentDirUrl . $archiveName);
162229
AJXP_Controller::applyHook("node.change", array(null, $newNode, false));
@@ -244,27 +311,55 @@ public function receiveAction($action, $httpVars, $fileVars)
244311
}
245312
mkdir($currentDirUrl . $onlyFileName, 0777, true);
246313
chmod(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName), 0777);
247-
try {
248-
$archive = new PharData(AJXP_MetaStreamWrapper::getRealFSReference($currentAllPydioPath));
249-
$fichiersArchive = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($pharCurrentAllPydioPath));
250-
foreach ($fichiersArchive as $file) {
251-
$fileGetPathName = $file->getPathname();
252-
if($file->isDir()) {
253-
continue;
314+
// Builds a command that will use any parallel extraction utilities installed to extract an archive file though tar.
315+
if (command_exists("tar")) {
316+
if ($pathInfoCurrentAllPydioPath == "gz") {
317+
if (command_exists("pigz")) {
318+
$tarcommand = "tar -I pigz -xf ";
319+
} else {
320+
$tarcommand = "tar -xzf ";
254321
}
255-
$fileNameInArchive = substr(strstr($fileGetPathName, $fileArchive), strlen($fileArchive) + 1);
256-
try {
257-
$archive->extractTo(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName), $fileNameInArchive, false);
258-
} catch (Exception $e) {
259-
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
260-
throw new AJXP_Exception($e);
322+
} elseif ($pathInfoCurrentAllPydioPath == "bz2") {
323+
if (command_exists("lbzip2")) {
324+
$tarcommand = "tar -I lbzip2 -xf ";
325+
} elseif (command_exists("pbzip2")) {
326+
$tarcommand = "tar -I pbzip2 -xf ";
327+
} else {
328+
$tarcommand = "tar -xjf ";
329+
}
330+
} elseif ($pathInfoCurrentAllPydioPath == "tar") {
331+
$tarcommand = "tar -xf ";
332+
} else {
333+
file_put_contents($progressExtractFileName, "Error : " . $messages["compression.15"]);
334+
throw new AJXP_Exception($messages["compression.15"]);
335+
}
336+
$extractCommand = $tarcommand . escapeshellarg(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $fileArchive)) . " -C " . escapeshellarg(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName));
337+
shell_exec($extractCommand);
338+
} else {
339+
// If tar command is not found, extract using PHP Phar instead
340+
try {
341+
$archive = new PharData(AJXP_MetaStreamWrapper::getRealFSReference($currentAllPydioPath));
342+
$fichiersArchive = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($pharCurrentAllPydioPath));
343+
foreach ($fichiersArchive as $file) {
344+
$fileGetPathName = $file->getPathname();
345+
if($file->isDir()) {
346+
continue;
347+
}
348+
$fileNameInArchive = substr(strstr($fileGetPathName, $fileArchive), strlen($fileArchive) + 1);
349+
try {
350+
$archive->extractTo(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName), $fileNameInArchive, false);
351+
} catch (Exception $e) {
352+
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
353+
throw new AJXP_Exception($e);
354+
}
355+
$counterExtract++;
356+
file_put_contents($progressExtractFileName, sprintf($messages["compression.13"], round(($counterExtract / $archive->count()) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
261357
}
262-
$counterExtract++;
263-
file_put_contents($progressExtractFileName, sprintf($messages["compression.13"], round(($counterExtract / $archive->count()) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
358+
} catch (Exception $e) {
359+
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
360+
throw new AJXP_Exception($e);
264361
}
265-
} catch (Exception $e) {
266-
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
267-
throw new AJXP_Exception($e);
362+
// End of Phar extraction section
268363
}
269364
file_put_contents($progressExtractFileName, "SUCCESS");
270365
$newNode = new AJXP_Node($currentDirUrl . $onlyFileName);
@@ -309,3 +404,4 @@ public function receiveAction($action, $httpVars, $fileVars)
309404
}
310405
}
311406
}
407+

0 commit comments

Comments
 (0)