diff --git a/.gitignore b/.gitignore index 8b7ef35..dfd6caa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ /vendor -composer.lock +composer.lock \ No newline at end of file diff --git a/README.md b/README.md index 9d7a9f2..83c2685 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ `b2-sdk-php` is a client library for working with Backblaze's B2 storage service. It aims to make using the service as easy as possible by exposing a clear API and taking influence from other SDKs that you may be familiar with. +This package will **cache authorization request** for 1 hour so that you won't receive API Limit from B2. + ## Example This is just a short example, full examples to come on the wiki. @@ -79,7 +81,7 @@ $fileList = $client->listFiles([ Installation is via Composer: ```bash -$ composer require cwhite92/b2-sdk-php +$ composer require runcloudio/b2-sdk-php ``` ## Tests diff --git a/composer.json b/composer.json index 3c80c9f..ee4df85 100644 --- a/composer.json +++ b/composer.json @@ -1,8 +1,8 @@ { - "name": "cwhite92/b2-sdk-php", - "description": "A SDK for working with B2 cloud storage.", + "name": "runcloudio/b2-sdk-php", + "description": "An SDK for working with B2 cloud storage.", "keywords": ["b2", "storage", "backblaze", "cloud", "filesystem", "backup"], - "homepage": "https://github.com/cwhite92/b2-sdk-php", + "homepage": "https://github.com/RunCloudIO/b2-sdk-php", "license": "MIT", "type": "library", "authors": [ @@ -10,11 +10,19 @@ "name": "Chris White", "email": "chris@cwhite.me", "homepage": "https://cwhite.me" + }, { + "name": "Ahmad Fikrizaman", + "email": "fikri@runcloud.io", + "homepage": "https://runcloud.io", + "role": "CTO" } ], "require": { "php": ">=5.5.0", - "guzzlehttp/guzzle": "^6.1" + "guzzlehttp/guzzle": "^6.1", + "illuminate/cache": ">=5.1.0", + "illuminate/container": ">=5.1.0", + "illuminate/filesystem": ">=5.1.0" }, "require-dev": { "phpunit/phpunit": "4.8.*" diff --git a/src/Cache/.gitignore b/src/Cache/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/src/Cache/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/src/Client.php b/src/Client.php index 55bbfd6..7e4267d 100644 --- a/src/Client.php +++ b/src/Client.php @@ -2,21 +2,36 @@ namespace ChrisWhite\B2; +use ChrisWhite\B2\Exceptions\CacheException; use ChrisWhite\B2\Exceptions\NotFoundException; use ChrisWhite\B2\Exceptions\ValidationException; use ChrisWhite\B2\Http\Client as HttpClient; +use Illuminate\Cache\CacheManager; +use Illuminate\Container\Container; +use Illuminate\Filesystem\Filesystem; class Client { protected $accountId; protected $applicationKey; + protected $cache; protected $authToken; protected $apiUrl; protected $downloadUrl; + protected $recommendedPartSize; protected $client; + /** + * Lower limit for using large files upload support. More information: + * https://www.backblaze.com/b2/docs/large_files.html. Default: 3 GB + * Files larger than this value will be uploaded in multiple parts. + * + * @var int + */ + protected $largeFileLimit = 3000000000; + /** * Client constructor. Accepts the account ID, application key and an optional array of options. * @@ -26,7 +41,7 @@ class Client */ public function __construct($accountId, $applicationKey, array $options = []) { - $this->accountId = $accountId; + $this->accountId = $accountId; $this->applicationKey = $applicationKey; if (isset($options['client'])) { @@ -35,9 +50,34 @@ public function __construct($accountId, $applicationKey, array $options = []) $this->client = new HttpClient(['exceptions' => false]); } + // initialize cache + $this->createCacheContainer(); + $this->authorizeAccount(); } + private function createCacheContainer() + { + $container = new Container; + $container['config'] = [ + 'cache.default' => 'file', + 'cache.stores.file' => [ + 'driver' => 'file', + 'path' => __DIR__ . '/Cache', + ], + ]; + $container['files'] = new Filesystem; + + try { + $cacheManager = new CacheManager($container); + $this->cache = $cacheManager->store(); + } catch (\Exception $e) { + throw new CacheException( + $e->getMessage() + ); + } + } + /** * Create a bucket with the given name and type. * @@ -53,15 +93,15 @@ public function createBucket(array $options) ); } - $response = $this->client->request('POST', $this->apiUrl.'/b2_create_bucket', [ + $response = $this->client->request('POST', $this->apiUrl . '/b2_create_bucket', [ 'headers' => [ 'Authorization' => $this->authToken, ], - 'json' => [ - 'accountId' => $this->accountId, + 'json' => [ + 'accountId' => $this->accountId, 'bucketName' => $options['BucketName'], - 'bucketType' => $options['BucketType'] - ] + 'bucketType' => $options['BucketType'], + ], ]); return new Bucket($response['bucketId'], $response['bucketName'], $response['bucketType']); @@ -86,15 +126,15 @@ public function updateBucket(array $options) $options['BucketId'] = $this->getBucketIdFromName($options['BucketName']); } - $response = $this->client->request('POST', $this->apiUrl.'/b2_update_bucket', [ + $response = $this->client->request('POST', $this->apiUrl . '/b2_update_bucket', [ 'headers' => [ 'Authorization' => $this->authToken, ], - 'json' => [ - 'accountId' => $this->accountId, - 'bucketId' => $options['BucketId'], - 'bucketType' => $options['BucketType'] - ] + 'json' => [ + 'accountId' => $this->accountId, + 'bucketId' => $options['BucketId'], + 'bucketType' => $options['BucketType'], + ], ]); return new Bucket($response['bucketId'], $response['bucketName'], $response['bucketType']); @@ -109,13 +149,13 @@ public function listBuckets() { $buckets = []; - $response = $this->client->request('POST', $this->apiUrl.'/b2_list_buckets', [ + $response = $this->client->request('POST', $this->apiUrl . '/b2_list_buckets', [ 'headers' => [ 'Authorization' => $this->authToken, ], - 'json' => [ - 'accountId' => $this->accountId - ] + 'json' => [ + 'accountId' => $this->accountId, + ], ]); foreach ($response['buckets'] as $bucket) { @@ -137,14 +177,14 @@ public function deleteBucket(array $options) $options['BucketId'] = $this->getBucketIdFromName($options['BucketName']); } - $this->client->request('POST', $this->apiUrl.'/b2_delete_bucket', [ + $this->client->request('POST', $this->apiUrl . '/b2_delete_bucket', [ 'headers' => [ - 'Authorization' => $this->authToken + 'Authorization' => $this->authToken, ], - 'json' => [ + 'json' => [ 'accountId' => $this->accountId, - 'bucketId' => $options['BucketId'] - ] + 'bucketId' => $options['BucketId'], + ], ]); return true; @@ -167,36 +207,6 @@ public function upload(array $options) $options['BucketId'] = $this->getBucketIdFromName($options['BucketName']); } - // Retrieve the URL that we should be uploading to. - $response = $this->client->request('POST', $this->apiUrl.'/b2_get_upload_url', [ - 'headers' => [ - 'Authorization' => $this->authToken - ], - 'json' => [ - 'bucketId' => $options['BucketId'] - ] - ]); - - $uploadEndpoint = $response['uploadUrl']; - $uploadAuthToken = $response['authorizationToken']; - - if (is_resource($options['Body'])) { - // We need to calculate the file's hash incrementally from the stream. - $context = hash_init('sha1'); - hash_update_stream($context, $options['Body']); - $hash = hash_final($context); - - // Similarly, we have to use fstat to get the size of the stream. - $size = fstat($options['Body'])['size']; - - // Rewind the stream before passing it to the HTTP client. - rewind($options['Body']); - } else { - // We've been given a simple string body, it's super simple to calculate the hash and size. - $hash = sha1($options['Body']); - $size = mb_strlen($options['Body']); - } - if (!isset($options['FileLastModified'])) { $options['FileLastModified'] = round(microtime(true) * 1000); } @@ -205,26 +215,13 @@ public function upload(array $options) $options['FileContentType'] = 'b2/x-auto'; } - $response = $this->client->request('POST', $uploadEndpoint, [ - 'headers' => [ - 'Authorization' => $uploadAuthToken, - 'Content-Type' => $options['FileContentType'], - 'Content-Length' => $size, - 'X-Bz-File-Name' => $options['FileName'], - 'X-Bz-Content-Sha1' => $hash, - 'X-Bz-Info-src_last_modified_millis' => $options['FileLastModified'] - ], - 'body' => $options['Body'] - ]); + list($options['hash'], $options['size']) = $this->getFileHashAndSize($options['Body']); - return new File( - $response['fileId'], - $response['fileName'], - $response['contentSha1'], - $response['contentLength'], - $response['contentType'], - $response['fileInfo'] - ); + if ($options['size'] <= $this->largeFileLimit && $options['size'] <= $this->recommendedPartSize) { + return $this->uploadStandardFile($options); + } else { + return $this->uploadLargeFile($options); + } } /** @@ -235,17 +232,17 @@ public function upload(array $options) */ public function download(array $options) { - $requestUrl = null; + $requestUrl = null; $requestOptions = [ 'headers' => [ - 'Authorization' => $this->authToken + 'Authorization' => $this->authToken, ], - 'sink' => isset($options['SaveAs']) ? $options['SaveAs'] : null + 'sink' => isset($options['SaveAs']) ? $options['SaveAs'] : fopen('php://temp', 'w'), ]; if (isset($options['FileId'])) { $requestOptions['query'] = ['fileId' => $options['FileId']]; - $requestUrl = $this->downloadUrl.'/b2api/v1/b2_download_file_by_id'; + $requestUrl = $this->downloadUrl . '/b2api/v1/b2_download_file_by_id'; } else { if (!isset($options['BucketName']) && isset($options['BucketId'])) { $options['BucketName'] = $this->getBucketNameFromId($options['BucketId']); @@ -254,11 +251,27 @@ public function download(array $options) $requestUrl = sprintf('%s/file/%s/%s', $this->downloadUrl, $options['BucketName'], $options['FileName']); } - $response = $this->client->request('GET', $requestUrl, $requestOptions, false); + if (isset($options['stream'])) { + $requestOptions['stream'] = $options['stream']; + $response = $this->client->request('GET', $requestUrl, $requestOptions, false, false); + } else { + $response = $this->client->request('GET', $requestUrl, $requestOptions, false); + } return isset($options['SaveAs']) ? true : $response; } + public function accelRedirectData(array $options) + { + $parsed = parse_url($this->downloadUrl); + + return [ + 'Authorization' => $this->authToken, + 'host' => $parsed['host'], + 'query' => sprintf("fileId=%s", $options['FileId']), + ]; + } + /** * Retrieve a collection of File objects representing the files stored inside a bucket. * @@ -272,7 +285,7 @@ public function listFiles(array $options) $nextFileName = null; $maxFileCount = 1000; - $files = []; + $files = []; if (!isset($options['BucketId']) && isset($options['BucketName'])) { $options['BucketId'] = $this->getBucketIdFromName($options['BucketName']); @@ -285,15 +298,15 @@ public function listFiles(array $options) // B2 returns, at most, 1000 files per "page". Loop through the pages and compile an array of File objects. while (true) { - $response = $this->client->request('POST', $this->apiUrl.'/b2_list_file_names', [ + $response = $this->client->request('POST', $this->apiUrl . '/b2_list_file_names', [ 'headers' => [ - 'Authorization' => $this->authToken + 'Authorization' => $this->authToken, ], - 'json' => [ - 'bucketId' => $options['BucketId'], + 'json' => [ + 'bucketId' => $options['BucketId'], 'startFileName' => $nextFileName, - 'maxFileCount' => $maxFileCount, - ] + 'maxFileCount' => $maxFileCount, + ], ]); foreach ($response['files'] as $file) { @@ -327,7 +340,6 @@ public function fileExists(array $options) return !empty($files); } - /** * Returns a single File object representing a file stored on B2. * @@ -345,13 +357,13 @@ public function getFile(array $options) } } - $response = $this->client->request('POST', $this->apiUrl.'/b2_get_file_info', [ + $response = $this->client->request('POST', $this->apiUrl . '/b2_get_file_info', [ 'headers' => [ - 'Authorization' => $this->authToken + 'Authorization' => $this->authToken, + ], + 'json' => [ + 'fileId' => $options['FileId'], ], - 'json' => [ - 'fileId' => $options['FileId'] - ] ]); return new File( @@ -387,14 +399,14 @@ public function deleteFile(array $options) $options['FileId'] = $file->getId(); } - $this->client->request('POST', $this->apiUrl.'/b2_delete_file_version', [ + $this->client->request('POST', $this->apiUrl . '/b2_delete_file_version', [ 'headers' => [ - 'Authorization' => $this->authToken + 'Authorization' => $this->authToken, ], - 'json' => [ + 'json' => [ 'fileName' => $options['FileName'], - 'fileId' => $options['FileId'] - ] + 'fileId' => $options['FileId'], + ], ]); return true; @@ -407,13 +419,20 @@ public function deleteFile(array $options) */ protected function authorizeAccount() { - $response = $this->client->request('GET', 'https://api.backblazeb2.com/b2api/v1/b2_authorize_account', [ - 'auth' => [$this->accountId, $this->applicationKey] - ]); + $client = $this->client; + $accountId = $this->accountId; + $applicationKey = $this->applicationKey; + + $response = $this->cache->remember('RunCloud-B2-SDK-Authorization', 60, function () use ($client, $accountId, $applicationKey) { + return $client->request('GET', 'https://api.backblazeb2.com/b2api/v1/b2_authorize_account', [ + 'auth' => [$accountId, $applicationKey], + ]); + }); - $this->authToken = $response['authorizationToken']; - $this->apiUrl = $response['apiUrl'].'/b2api/v1'; - $this->downloadUrl = $response['downloadUrl']; + $this->authToken = $response['authorizationToken']; + $this->apiUrl = $response['apiUrl'] . '/b2api/v1'; + $this->downloadUrl = $response['downloadUrl']; + $this->recommendedPartSize = $response['recommendedPartSize']; } /** @@ -458,7 +477,7 @@ protected function getFileIdFromBucketAndFileName($bucketName, $fileName) { $files = $this->listFiles([ 'BucketName' => $bucketName, - 'FileName' => $fileName, + 'FileName' => $fileName, ]); foreach ($files as $file) { @@ -469,4 +488,187 @@ protected function getFileIdFromBucketAndFileName($bucketName, $fileName) return null; } + + /** + * Calculate hash and size of file/stream. If $offset and $partSize is given return + * hash and size of this chunk + * + * @param $content + * @param int $offset + * @param null $partSize + * @return array + */ + protected function getFileHashAndSize($data, $offset = 0, $partSize = null) + { + if (!$partSize) { + if (is_resource($data)) { + // We need to calculate the file's hash incrementally from the stream. + $context = hash_init('sha1'); + hash_update_stream($context, $data); + $hash = hash_final($context); + // Similarly, we have to use fstat to get the size of the stream. + $size = fstat($data)['size']; + // Rewind the stream before passing it to the HTTP client. + rewind($data); + } else { + // We've been given a simple string body, it's super simple to calculate the hash and size. + $hash = sha1($data); + $size = mb_strlen($data, '8bit'); + } + } else { + $dataPart = $this->getPartOfFile($data, $offset, $partSize); + $hash = sha1($dataPart); + $size = mb_strlen($dataPart, '8bit'); + } + + return array($hash, $size); + } + + /** + * Return selected part of file + * + * @param $data + * @param $offset + * @param $partSize + * @return bool|string + */ + protected function getPartOfFile($data, $offset, $partSize) + { + // Get size and hash of one data chunk + if (is_resource($data)) { + // Get data chunk + fseek($data, $offset); + $dataPart = fread($data, $partSize); + // Rewind the stream before passing it to the HTTP client. + rewind($data); + } else { + $dataPart = substr($data, $offset, $partSize); + } + return $dataPart; + } + + /** + * Upload single file (smaller than 3 GB) + * + * @param array $options + * @return File + */ + protected function uploadStandardFile($options = array()) + { + // Retrieve the URL that we should be uploading to. + $response = $this->client->request('POST', $this->apiUrl . '/b2_get_upload_url', [ + 'headers' => [ + 'Authorization' => $this->authToken, + ], + 'json' => [ + 'bucketId' => $options['BucketId'], + ], + ]); + + $uploadEndpoint = $response['uploadUrl']; + $uploadAuthToken = $response['authorizationToken']; + + $response = $this->client->request('POST', $uploadEndpoint, [ + 'headers' => [ + 'Authorization' => $uploadAuthToken, + 'Content-Type' => $options['FileContentType'], + 'Content-Length' => $options['size'], + 'X-Bz-File-Name' => $options['FileName'], + 'X-Bz-Content-Sha1' => $options['hash'], + 'X-Bz-Info-src_last_modified_millis' => $options['FileLastModified'], + ], + 'body' => $options['Body'], + ]); + + return new File( + $response['fileId'], + $response['fileName'], + $response['contentSha1'], + $response['contentLength'], + $response['contentType'], + $response['fileInfo'] + ); + } + + /** + * Upload large file. Large files will be uploaded in chunks of recommendedPartSize bytes (usually 100MB each) + * + * @param array $options + * @return File + */ + protected function uploadLargeFile($options) + { + // Prepare for uploading the parts of a large file. + $response = $this->client->request('POST', $this->apiUrl . '/b2_start_large_file', [ + 'headers' => [ + 'Authorization' => $this->authToken, + ], + 'json' => [ + 'bucketId' => $options['BucketId'], + 'fileName' => $options['FileName'], + 'contentType' => $options['FileContentType'], + /** + 'fileInfo' => [ + 'src_last_modified_millis' => $options['FileLastModified'], + 'large_file_sha1' => $options['hash'] + ] + **/ + ], + ]); + $fileId = $response['fileId']; + + $partsCount = ceil($options['size'] / $this->recommendedPartSize); + + $hashParts = []; + for ($i = 1; $i <= $partsCount; $i++) { + $bytesSent = ($i - 1) * $this->recommendedPartSize; + $bytesLeft = $options['size'] - $bytesSent; + $partSize = ($bytesLeft > $this->recommendedPartSize) ? $this->recommendedPartSize : $bytesLeft; + + // Retrieve the URL that we should be uploading to. + $response = $this->client->request('POST', $this->apiUrl . '/b2_get_upload_part_url', [ + 'headers' => [ + 'Authorization' => $this->authToken, + ], + 'json' => [ + 'fileId' => $fileId, + ], + ]); + + $uploadEndpoint = $response['uploadUrl']; + $uploadAuthToken = $response['authorizationToken']; + + list($hash, $size) = $this->getFileHashAndSize($options['Body'], $bytesSent, $partSize); + $hashParts[] = $hash; + + $response = $this->client->request('POST', $uploadEndpoint, [ + 'headers' => [ + 'Authorization' => $uploadAuthToken, + 'X-Bz-Part-Number' => $i, + 'Content-Length' => $size, + 'X-Bz-Content-Sha1' => $hash, + ], + 'body' => $this->getPartOfFile($options['Body'], $bytesSent, $partSize), + ]); + } + + // Finish upload of large file + $response = $this->client->request('POST', $this->apiUrl . '/b2_finish_large_file', [ + 'headers' => [ + 'Authorization' => $this->authToken, + ], + 'json' => [ + 'fileId' => $fileId, + 'partSha1Array' => $hashParts, + ], + ]); + return new File( + $response['fileId'], + $response['fileName'], + $response['contentSha1'], + $response['contentLength'], + $response['contentType'], + $response['fileInfo'] + ); + } } diff --git a/src/ErrorHandler.php b/src/ErrorHandler.php index 54ac06c..2fc79ee 100644 --- a/src/ErrorHandler.php +++ b/src/ErrorHandler.php @@ -6,20 +6,20 @@ use ChrisWhite\B2\Exceptions\BadJsonException; use ChrisWhite\B2\Exceptions\BadValueException; use ChrisWhite\B2\Exceptions\BucketAlreadyExistsException; -use ChrisWhite\B2\Exceptions\NotFoundException; -use ChrisWhite\B2\Exceptions\FileNotPresentException; use ChrisWhite\B2\Exceptions\BucketNotEmptyException; +use ChrisWhite\B2\Exceptions\FileNotPresentException; +use ChrisWhite\B2\Exceptions\NotFoundException; use GuzzleHttp\Psr7\Response; class ErrorHandler { protected static $mappings = [ - 'bad_json' => BadJsonException::class, - 'bad_value' => BadValueException::class, - 'duplicate_bucket_name' => BucketAlreadyExistsException::class, - 'not_found' => NotFoundException::class, - 'file_not_present' => FileNotPresentException::class, - 'cannot_delete_non_empty_bucket' => BucketNotEmptyException::class + 'bad_json' => BadJsonException::class, + 'bad_value' => BadValueException::class, + 'duplicate_bucket_name' => BucketAlreadyExistsException::class, + 'not_found' => NotFoundException::class, + 'file_not_present' => FileNotPresentException::class, + 'cannot_delete_non_empty_bucket' => BucketNotEmptyException::class, ]; public static function handleErrorResponse(Response $response) @@ -33,6 +33,6 @@ public static function handleErrorResponse(Response $response) $exceptionClass = B2Exception::class; } - throw new $exceptionClass('Received error from B2: '.$responseJson['message']); + throw new $exceptionClass(sprintf('Received error from B2: %s. Code: %s', $responseJson['message'], $responseJson['code'])); } } diff --git a/src/Exceptions/CacheException.php b/src/Exceptions/CacheException.php new file mode 100644 index 0000000..184598e --- /dev/null +++ b/src/Exceptions/CacheException.php @@ -0,0 +1,7 @@ +retryWaitSec; + while ($response->getStatusCode() === 503 and $this->retryLimit > $retries) { + $retries++; + sleep($wait); + $response = parent::request($method, $uri, $options); + // Wait 20% longer if it fails again + $wait *= 1.2; + } if ($response->getStatusCode() !== 200) { ErrorHandler::handleErrorResponse($response); } @@ -33,6 +46,10 @@ public function request($method, $uri = null, array $options = [], $asJson = tru return json_decode($response->getBody(), true); } - return $response->getBody()->getContents(); + if (!$wantsGetContents) { + return $response->getBody(); + } + + return $response->getBody(); } } diff --git a/tests/responses/authorize_account.json b/tests/responses/authorize_account.json index f1a925d..e5510c8 100644 --- a/tests/responses/authorize_account.json +++ b/tests/responses/authorize_account.json @@ -2,5 +2,6 @@ "accountId": "testId", "apiUrl": "https://api900.backblaze.com", "authorizationToken": "testAuthToken", - "downloadUrl": "https://f900.backblaze.com" + "downloadUrl": "https://f900.backblaze.com", + "recommendedPartSize": 100000000 } \ No newline at end of file