Skip to content

Commit ba16ed4

Browse files
camillebaronnetcamillebaronnet
committed
Add Http extractor, #1 
1 parent 42402ef commit ba16ed4

File tree

10 files changed

+363
-30
lines changed

10 files changed

+363
-30
lines changed

‎README.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,12 @@ use Camillebaronnet\ETL\Loader\Json;
2929
//...
3030

3131
$etl = (new LayerStrategy)
32-
->extract(Csv::class, ['filename' => 'dump.php'])
33-
->transform(Trim::class)
32+
->extract(Http::class, [
33+
'url' => 'https://api.github.com/users/camillebaronnet/repos'
34+
])
35+
->transform(Flatten::class, [
36+
'glue' => '_'
37+
])
3438
->transform(DateTime::class, ['format' => 'd/m/Y', 'fields' => ['createAt']])
3539
;
3640

‎src/Exception/DecoderNotFound.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?php
2+
3+
namespace Camillebaronnet\ETL\Exception;
4+
5+
use Exception;
6+
use Throwable;
7+
8+
class DecoderNotFound extends Exception
9+
{
10+
public function __construct(string $message = 'Decoder not found.', int $code = 0, Throwable $previous = null)
11+
{
12+
parent::__construct($message, $code, $previous);
13+
}
14+
}

‎src/Extractor/AbstractExtractor.php‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,21 @@
33
namespace Camillebaronnet\ETL\Extractor;
44

55
use Camillebaronnet\ETL\Exception\MissingParameter;
6+
use Camillebaronnet\ETL\Extractor\Extension\SupportDecoders;
67

78
abstract class AbstractExtractor implements ExtractInterface
89
{
10+
use SupportDecoders;
11+
912
/**
1013
* @param array $requiredFields
1114
* @param array $params
1215
* @throws MissingParameter
1316
*/
14-
public function requiredParameters(array $requiredFields, array $params)
17+
protected function requiredParameters(array $requiredFields, array $params)
1518
{
16-
foreach($requiredFields as $field){
17-
if(!isset($params[$field])){
19+
foreach ($requiredFields as $field) {
20+
if (!isset($params[$field])) {
1821
throw new MissingParameter('Parameter "'.$field.'" is missing.');
1922
}
2023
}

‎src/Extractor/Csv.php‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎src/Extractor/Extension/SupportDecoders.php‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
<?php
2+
3+
namespace Camillebaronnet\ETL\Extractor\Extension;
4+
5+
use Camillebaronnet\ETL\Exception\BadInterface;
6+
use Camillebaronnet\ETL\Exception\DecoderNotFound;
7+
use Symfony\Component\Serializer\Encoder\CsvEncoder;
8+
use Symfony\Component\Serializer\Encoder\DecoderInterface;
9+
use Symfony\Component\Serializer\Encoder\JsonEncoder;
10+
use Symfony\Component\Serializer\Encoder\XmlEncoder;
11+
use Symfony\Component\Serializer\Encoder\YamlEncoder;
12+
13+
trait SupportDecoders
14+
{
15+
/**
16+
* Content-Type decoders matches.
17+
*/
18+
protected static $TYPE_MIME_DECODERS = [
19+
'application/json' => JsonEncoder::class,
20+
'application/xml' => XmlEncoder::class,
21+
'text/yaml' => YamlEncoder::class,
22+
'text/x-yaml' => YamlEncoder::class,
23+
'application/yaml' => YamlEncoder::class,
24+
'text/vnd.yaml' => YamlEncoder::class,
25+
'application/x-yaml' => YamlEncoder::class,
26+
'text/csv' => CsvEncoder::class,
27+
];
28+
29+
/**
30+
* Content-Type extension decoders matches.
31+
*/
32+
protected static $EXTENSIONS_DECODERS = [
33+
'json' => JsonEncoder::class,
34+
'xml' => XmlEncoder::class,
35+
'yaml' => YamlEncoder::class,
36+
'yml' => YamlEncoder::class,
37+
'csv' => CsvEncoder::class,
38+
];
39+
40+
/**
41+
* Extends the support of more type mime.
42+
*
43+
* @param $typeMime
44+
* @param $decoderClass
45+
* @throws BadInterface
46+
*/
47+
public static function setTypeMimeDecoder($typeMime, $decoderClass): void
48+
{
49+
if (!new $decoderClass instanceof DecoderInterface) {
50+
throw new BadInterface(sprintf('%s must be an instance of DecoderInterface.', $decoderClass));
51+
}
52+
53+
static::$TYPE_MIME_DECODERS[$typeMime] = $decoderClass;
54+
}
55+
56+
/**
57+
* Extends the support for more extensions.
58+
*
59+
* @param $extension
60+
* @param $decoderClass
61+
* @throws BadInterface
62+
*/
63+
public static function setExtensionDecoder($extension, $decoderClass): void
64+
{
65+
if (!new $decoderClass instanceof DecoderInterface) {
66+
throw new BadInterface(sprintf('%s must be an instance of DecoderInterface.', $decoderClass));
67+
}
68+
69+
static::$EXTENSIONS_DECODERS[$extension] = $decoderClass;
70+
}
71+
72+
/**
73+
* @param $contentType
74+
* @return null
75+
*/
76+
protected function findDecoder($contentType)
77+
{
78+
preg_match('/^[^;]+/', $contentType, $matches);
79+
$contentType = trim($matches[0] ?? null);
80+
81+
if (isset(static::$TYPE_MIME_DECODERS[$contentType])) {
82+
return static::$TYPE_MIME_DECODERS[$contentType];
83+
}
84+
85+
preg_match('/\+([^+]+)$/', $contentType, $matches);
86+
$contentTypeExtension = $matches[1] ?? null;
87+
88+
return static::$EXTENSIONS_DECODERS[$contentTypeExtension] ?? null;
89+
}
90+
91+
/**
92+
* @param $body
93+
* @param $typeMime
94+
* @param array $context
95+
* @return mixed
96+
* @throws BadInterface
97+
* @throws DecoderNotFound
98+
*/
99+
protected function decode($body, $typeMime, $context = [])
100+
{
101+
$contextDecoder = $context['decoder'] ?? [];
102+
$decoderClass = $contextDecoder['class'] ?? $this->findDecoder($typeMime);
103+
104+
if (null === $decoderClass) {
105+
throw new DecoderNotFound;
106+
}
107+
108+
$decoder = new $decoderClass;
109+
110+
if (!$decoder instanceof DecoderInterface) {
111+
throw new BadInterface(sprintf('Bad interface. %s must be an instance of DecoderInterface.',
112+
$decoderClass));
113+
}
114+
115+
$format = defined($decoderClass.'::FORMAT') ? $decoderClass::FORMAT : null;
116+
117+
return $decoder->decode($body, $format, $context);
118+
}
119+
}

‎src/Extractor/Http.php‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?php
2+
3+
namespace Camillebaronnet\ETL\Extractor;
4+
5+
use Camillebaronnet\ETL\Exception\BadInterface;
6+
use Camillebaronnet\ETL\Exception\MissingParameter;
7+
8+
class Http extends AbstractExtractor
9+
{
10+
/**
11+
* Default context.
12+
*/
13+
public const DEFAULT_CONTEXT = [
14+
'method' => 'GET',
15+
'data' => null,
16+
'headers' => [],
17+
'curl_opts' => [
18+
CURLOPT_RETURNTRANSFER => true,
19+
CURLOPT_FOLLOWLOCATION => true,
20+
CURLOPT_USERAGENT => '-',
21+
CURLOPT_AUTOREFERER => true,
22+
CURLOPT_SSL_VERIFYHOST => 2,
23+
CURLOPT_SSL_VERIFYPEER => true,
24+
],
25+
];
26+
27+
/**
28+
* @param array $context
29+
* @return iterable
30+
* @throws MissingParameter
31+
* @throws BadInterface
32+
*/
33+
public function __invoke(array $context = []): iterable
34+
{
35+
$this->requiredParameters(['url'], $context);
36+
37+
$context = array_merge(static::DEFAULT_CONTEXT, $context);
38+
$curl_opts = [
39+
CURLOPT_URL => $context['url'],
40+
CURLOPT_CUSTOMREQUEST => strtoupper($context['method']),
41+
];
42+
43+
if (null !== $context['data']) {
44+
$curl_opts += [
45+
CURLOPT_POST => true,
46+
CURLOPT_POSTFIELDS => $context['data'],
47+
];
48+
}
49+
50+
if ('HEAD' === strtoupper($context['method'])) {
51+
$curl_opts += [CURLOPT_NOBODY => 1];
52+
}
53+
54+
$ch = curl_init();
55+
curl_setopt_array($ch, $curl_opts + $context['curl_opts']);
56+
$content = curl_exec($ch);
57+
$contentType = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
58+
curl_close($ch);
59+
60+
return $this->decode(
61+
$content,
62+
$contentType,
63+
$context
64+
);
65+
}
66+
}

‎src/Tests/Extractor/SupportDecodersTest.php‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
<?php
2+
3+
namespace Camillebaronnet\ETL\Tests\Strategy;
4+
5+
use Camillebaronnet\ETL\Exception\BadInterface;
6+
use Camillebaronnet\ETL\Exception\DecoderNotFound;
7+
use Camillebaronnet\ETL\Tests\Fixtures\DummyDecoder;
8+
use Camillebaronnet\ETL\Tests\Fixtures\SupportDecoderProxy;
9+
use PHPUnit\Framework\TestCase;
10+
11+
final class SupportDecodersTest extends TestCase
12+
{
13+
/**
14+
* @var SupportDecoderProxy
15+
*/
16+
private $extractor;
17+
18+
protected function setUp()
19+
{
20+
$this->extractor = new SupportDecoderProxy();
21+
}
22+
23+
/**
24+
* @throws BadInterface
25+
* @throws DecoderNotFound
26+
*/
27+
public function testExistingFormatCanByParsed()
28+
{
29+
$data = ['foo' => 'bar'];
30+
31+
$decodedData = $this->extractor->decodeHandler(
32+
json_encode($data),
33+
'application/json',
34+
[]
35+
);
36+
37+
$this->assertEquals($decodedData, $data);
38+
}
39+
40+
/**
41+
* @throws BadInterface
42+
* @throws DecoderNotFound
43+
*/
44+
public function testUnknowFormatThrowsAnException()
45+
{
46+
$this->expectException(DecoderNotFound::class);
47+
$this->extractor->decodeHandler(
48+
'...',
49+
'wingardium leviosa',
50+
[]
51+
);
52+
}
53+
54+
/**
55+
* @throws BadInterface
56+
* @throws DecoderNotFound
57+
*/
58+
public function testExtendDecoderByPassingANewTypeMimeWorks()
59+
{
60+
$data = ['foo' => 'bar'];
61+
SupportDecoderProxy::setTypeMimeDecoder('application/custom', DummyDecoder::class);
62+
$decodedData = $this->extractor->decodeHandler(
63+
$data,
64+
'application/custom',
65+
[]
66+
);
67+
68+
$this->assertEquals($decodedData, $data);
69+
}
70+
71+
/**
72+
* @throws BadInterface
73+
* @throws DecoderNotFound
74+
*/
75+
public function testExtendDecoderByPassingAnExtensionWorks()
76+
{
77+
$data = ['foo' => 'bar'];
78+
SupportDecoderProxy::setExtensionDecoder('custom', DummyDecoder::class);
79+
$decodedData = $this->extractor->decodeHandler(
80+
$data,
81+
'application/random+custom',
82+
[]
83+
);
84+
85+
$this->assertEquals($decodedData, $data);
86+
}
87+
88+
/**
89+
* @throws BadInterface
90+
* @throws DecoderNotFound
91+
*/
92+
public function testForceDecoderByContextVariable()
93+
{
94+
$data = ['foo' => 'bar'];
95+
$decodedData = $this->extractor->decodeHandler(
96+
$data,
97+
'application/random+custom',
98+
[
99+
'decoder' => [
100+
'class' => DummyDecoder::class,
101+
],
102+
]
103+
);
104+
105+
$this->assertEquals($decodedData, $data);
106+
}
107+
}

‎src/Tests/Fixtures/DummyDecoder.php‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?php
2+
3+
namespace Camillebaronnet\ETL\Tests\Fixtures;
4+
5+
use Symfony\Component\Serializer\Encoder\DecoderInterface;
6+
7+
class DummyDecoder implements DecoderInterface
8+
{
9+
/**
10+
* {@inheritdoc}
11+
*/
12+
public function supportsDecoding($format)
13+
{
14+
return true;
15+
}
16+
17+
/**
18+
* {@inheritdoc}
19+
*/
20+
public function decode($data, $format, array $context = [])
21+
{
22+
return $data;
23+
}
24+
}

0 commit comments

Comments
 (0)