Skip to content

Commit 9e36683

Browse files
committed
Use parallel gunzip process for faster gzip decompression
1 parent e829f5e commit 9e36683

File tree

2 files changed

+14
-10
lines changed

2 files changed

+14
-10
lines changed

composer.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
"react/stream": "^1.0 || ^0.7 || ^0.6"
1919
},
2020
"require-dev": {
21-
"clue/zlib-react": "^1.0 || ^0.2.2",
2221
"phpunit/phpunit": "^7.0 || ^6.0 || ^5.7 || ^4.8.35",
22+
"react/child-process": "^0.6",
2323
"react/event-loop": "^1.0 || ^0.5 || ^0.4 || ^0.3"
2424
}
2525
}

examples/92-benchmark-count-gzip.php

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,8 @@
1111
// $ php examples/92-benchmark-count-gzip.php < IRAhandle_tweets_1.csv.gz
1212

1313
use Clue\React\Csv\AssocDecoder;
14-
use Clue\React\Zlib\Decompressor;
14+
use React\ChildProcess\Process;
1515
use React\EventLoop\Factory;
16-
use React\Stream\ReadableResourceStream;
1716

1817
require __DIR__ . '/../vendor/autoload.php';
1918

@@ -22,14 +21,19 @@
2221
}
2322

2423
$loop = Factory::create();
25-
$input = new ReadableResourceStream(STDIN, $loop);
26-
$decompressor = new Decompressor(ZLIB_ENCODING_GZIP);
27-
$input->pipe($decompressor);
28-
$decoder = new AssocDecoder($decompressor);
2924

30-
$decompressor->on('error', function (Exception $e) {
31-
printf("\nDecompression error: " . $e->getMessage() . "\n");
32-
});
25+
// This benchmark example spawns the decompressor in a child `gunzip` process
26+
// because parsing CSV files is already mostly CPU-bound and multi-processing
27+
// is preferred here. If the input source is slower (such as an HTTP download)
28+
// or if `gunzip` is not available (Windows), using a built-in decompressor
29+
// such as https://github.com/clue/reactphp-zlib would be preferable.
30+
$process = new Process('exec gunzip', null, null, array(
31+
0 => STDIN,
32+
1 => array('pipe', 'w'),
33+
STDERR
34+
));
35+
$process->start($loop);
36+
$decoder = new AssocDecoder($process->stdout);
3337

3438
$count = 0;
3539
$decoder->on('data', function () use (&$count) {

0 commit comments

Comments
 (0)