diff --git a/README.md b/README.md index 0695080..36a415d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ php-WebHDFS is a PHP client for [WebHDFS](http://hadoop.apache.org/docs/r2.0.3-a ## Install via composer ```bash -composer require simpleenergy/php-webhdfs +composer require dreamfactory/php-webhdfs ``` ## Usage @@ -149,4 +149,4 @@ $hdfs->setReplication('user/hadoop-username/file.txt', '2'); ```php $hdfs = new WebHDFS('mynamenode.hadoop.com', '50070', 'hadoop-username'); $response = $hdfs->setTimes('user/hadoop-username/file.txt'); -``` \ No newline at end of file +``` diff --git a/composer.json b/composer.json index 85baaaf..3bfdc02 100644 --- a/composer.json +++ b/composer.json @@ -1,9 +1,14 @@ { - "name": "simpleenergy/php-webhdfs", + "name": "dreamfactory/php-webhdfs", "description": "PHP WebHDFS, forked from https://github.com/simpleenergy/php-WebHDFS", "minimum-stability": "stable", + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/dreamfactorysoftware/php-WebHDFS" + } + ], "license": "MIT", - "version": "1.0.7", "authors": [ { "name": "tranch-xiao", @@ -28,7 +33,8 @@ ], "require": { "php": ">=5.4.0", - "ext-curl": "*" + "ext-curl": "*", + "ext-json": "*" }, "autoload": { "psr-0": { diff --git a/src/org/apache/hadoop/WebHDFS.php b/src/org/apache/hadoop/WebHDFS.php index fe664ef..dee95b0 100644 --- a/src/org/apache/hadoop/WebHDFS.php +++ b/src/org/apache/hadoop/WebHDFS.php @@ -52,7 +52,7 @@ public function create( $options = array( 'op' => 'CREATE', - 'overwrite' => $overwrite, + 'overwrite' => $overwrite ? 'true' : 'false', 'blocksize' => $blocksize, 'replication' => $replication, 'permission' => $permission, @@ -79,7 +79,7 @@ public function createWithData( ) { $options = array( 'op' => 'CREATE', - 'overwrite' => $overwrite, + 'overwrite' => $overwrite ? 'true' : 'false', 'blocksize' => $blockSize, 'replication' => $replication, 'permission' => $permission, @@ -200,7 +200,15 @@ private function _listStatus($path, $cleanLastRequest = false) } } - public function listFiles($path, $recursive = false, $includeFileMetaData = false, $maxAmountOfFiles = false) + /** + * @param string $path + * @param bool $recursive + * @param bool $includeFileMetaData + * @param int $maxDepth max depth to search recursively. When value below zero, it will search to the end of tree + * @return array + * @throws WebHDFS_Exception + */ + public function listFiles($path, $recursive = false, $includeFileMetaData = false, $maxDepth = -1) { $result = array(); $listStatusResult = $this->_listStatus($path); @@ -211,21 +219,27 @@ public function listFiles($path, $recursive = false, $includeFileMetaData = fals switch ($fileEntity->type) { case 'DIRECTORY': if ($recursive === true) { - $result = array_merge($result, - $this->listFiles($path.$fileEntity->pathSuffix.'/', true, $includeFileMetaData, - $maxAmountOfFiles - sizeof($result))); + $result = array_merge( + $result, + $this->listFiles( + $this->concatPath([$path, $fileEntity->pathSuffix]), + true, + $includeFileMetaData, + $maxDepth - 1 + ) + ); } break; default: if ($includeFileMetaData === true) { - $fileEntity->path = $path.$fileEntity->pathSuffix; + $fileEntity->path = $this->concatPath([$path, $fileEntity->pathSuffix]); $result[] = $fileEntity; } else { - $result[] = $path.$fileEntity->pathSuffix; + $result[] = $this->concatPath([$path, $fileEntity->pathSuffix]); } } // recursion will be interrupted since we subtract the amount of the current result set from the maxAmountOfFiles amount with calling the next recursion - if (sizeof($result) >= $maxAmountOfFiles) { + if ($maxDepth === 0) { break; } } @@ -247,15 +261,19 @@ public function listDirectories($path, $recursive = false, $includeFileMetaData switch ($fileEntity->type) { case 'DIRECTORY': if ($includeFileMetaData === true) { - $fileEntity->path = $path.$fileEntity->pathSuffix; + $fileEntity->path = $this->concatPath([$path, $fileEntity->pathSuffix]); $result[] = $fileEntity; } else { - $result[] = $path.$fileEntity->pathSuffix; + $result[] = $this->concatPath([$path, $fileEntity->pathSuffix]); } if ($recursive === true) { $result = array_merge($result, - $this->listDirectories($path.$fileEntity->pathSuffix.'/', $recursive, - $includeFileMetaData)); + $this->listDirectories( + $this->concatPath([$path, $fileEntity->pathSuffix]), + $recursive, + $includeFileMetaData + ) + ); } break; } @@ -445,4 +463,22 @@ private function getResponseErrorException($responseData) return new WebHDFS_Exception($exceptionMessage, $exceptionCode); } + + private function concatPath(array $paths) { + $result = ''; + foreach ($paths as $path) { + if (!$result || preg_match('/.+\/$/', $result)) { + $result .= $path; + continue; + } + + $result .= '/' . $path; + } + return $this->removeMultiSlashFromPath($result); + } + + private function removeMultiSlashFromPath($path) { + return preg_replace('/(\/)\/+/', '$1', $path); + } + } diff --git a/src/org/apache/hadoop/tools/Curl.php b/src/org/apache/hadoop/tools/Curl.php index 327d5e9..190a270 100644 --- a/src/org/apache/hadoop/tools/Curl.php +++ b/src/org/apache/hadoop/tools/Curl.php @@ -63,7 +63,6 @@ private function _findRedirectUrl($url, $options) { $options[CURLOPT_URL] = $url; $options[CURLOPT_HEADER] = true; - $options[CURLINFO_EFFECTIVE_URL] = true; $options[CURLOPT_RETURNTRANSFER] = true; $header = $this->_exec($options); $matches = array(); @@ -221,8 +220,6 @@ private function _exec($options, $returnInfo = false) ['Content-Length: '.strlen($options[CURLOPT_POSTFIELDS])] ); } - } else { - $options[CURLOPT_HTTPHEADER] = array_merge($options[CURLOPT_HTTPHEADER], ['Content-Length: 0']); } }