Skip to content

Commit 4cce83f

Browse files
stloydstloyd
authored
[Http] Add Http adapter functions (#1911)
* [Http] Add Http adapter functions Avro & Excel added to DSL documentation * Update Rector configuration & re-run it * Remove multiple path support in `from_avro()` function Don't validate it due to lack of support
1 parent 49734e3 commit 4cce83f

File tree

32 files changed

+169
-116
lines changed

32 files changed

+169
-116
lines changed

‎bin/docs.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,14 @@ public function execute(InputInterface $input, OutputInterface $output) : int
3737

3838
$paths = [
3939
__DIR__ . '/../src/core/etl/src/Flow/ETL/DSL/functions.php',
40+
__DIR__ . '/../src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php',
4041
__DIR__ . '/../src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php',
4142
__DIR__ . '/../src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php',
4243
__DIR__ . '/../src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/functions.php',
44+
__DIR__ . '/../src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/DSL/functions.php',
4345
__DIR__ . '/../src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/functions.php',
4446
__DIR__ . '/../src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php',
47+
__DIR__ . '/../src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/DSL/functions.php',
4548
__DIR__ . '/../src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php',
4649
__DIR__ . '/../src/adapter/etl-adapter-meilisearch/src/Flow/ETL/Adapter/Meilisearch/functions.php',
4750
__DIR__ . '/../src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php',

‎composer.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@
146146
"src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/functions.php",
147147
"src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/DSL/functions.php",
148148
"src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php",
149+
"src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/DSL/functions.php",
149150
"src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php",
150151
"src/adapter/etl-adapter-meilisearch/src/Flow/ETL/Adapter/Meilisearch/functions.php",
151152
"src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php",

‎phpstan.neon‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ parameters:
6161
- src/cli/src/Flow/CLI/Command/*
6262
- src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIITable.php
6363
- src/core/etl/src/Flow/ETL/Sort/ExternalSort/RowsMinHeap.php
64+
- src/adapter/etl-adapter-avro/*
6465
- src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/ElasticsearchPHP/SearchResults.php
6566
- src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/ElasticsearchPHP/SearchParams.php
6667
- src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/ElasticsearchPHP/PointInTime.php

‎rector.src.php‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@
1414
__DIR__ . '/src/core/etl/src',
1515
__DIR__ . '/src/cli/src',
1616
__DIR__ . '/src/lib/*/src',
17-
__DIR__ . '/src/adapter/*/src',
18-
__DIR__ . '/src/bridge/*/*/src',
19-
__DIR__ . '/src/tools/*/*/src',
17+
__DIR__ . '/src/adapter/**/src',
18+
__DIR__ . '/src/bridge/**/src',
19+
__DIR__ . '/src/tools/**/src',
2020
])
2121
->withSkip([
2222
RemoveExtraParametersRector::class,
@@ -28,5 +28,5 @@
2828
->withCache(__DIR__ . '/var/rector/src')
2929
->withImportNames(importShortClasses: false, removeUnusedImports: true)
3030
->withSets([
31-
LevelSetList::UP_TO_PHP_82
32-
]);
31+
LevelSetList::UP_TO_PHP_82,
32+
]);

‎rector.tests.php‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
11
<?php
22

3+
use Flow\ETL\Adapter\Avro\FlixTech\AvroExtractor;
4+
use Flow\ETL\Adapter\CSV\CSVExtractor;
5+
use Flow\ETL\Adapter\Doctrine\DbalQueryExtractor;
6+
use Flow\ETL\Adapter\Elasticsearch\ElasticsearchPHP\ElasticsearchExtractor;
7+
use Flow\ETL\Adapter\Excel\ExcelExtractor;
8+
use Flow\ETL\Adapter\GoogleSheet\GoogleSheetExtractor;
9+
use Flow\ETL\Adapter\Http\PsrHttpClientDynamicExtractor;
10+
use Flow\ETL\Adapter\Http\PsrHttpClientStaticExtractor;
11+
use Flow\ETL\Adapter\JSON\JSONMachine\JsonExtractor;
12+
use Flow\ETL\Adapter\JSON\JSONMachine\JsonLinesExtractor;
13+
use Flow\ETL\Adapter\Meilisearch\MeilisearchPHP\MeilisearchExtractor;
14+
use Flow\ETL\Adapter\Parquet\ParquetExtractor;
15+
use Flow\ETL\Adapter\Text\TextExtractor;
16+
use Flow\ETL\Adapter\XML\XMLParserExtractor;
317
use Flow\ETL\Config;
418
use Flow\ETL\Extractor\ArrayExtractor;
519
use Flow\ETL\Extractor\CacheExtractor;
@@ -67,12 +81,12 @@
6781
__DIR__ . '/src/core/etl/tests',
6882
__DIR__ . '/src/cli/tests',
6983
__DIR__ . '/src/lib/*/tests',
70-
__DIR__ . '/src/adapter/*/tests',
71-
__DIR__ . '/src/bridge/*/*/tests',
72-
__DIR__ . '/src/tools/*/*/tests',
84+
__DIR__ . '/src/adapter/**/tests',
85+
__DIR__ . '/src/bridge/**/tests',
86+
__DIR__ . '/src/tools/**/tests',
7387
])
7488
->withSets([
75-
LevelSetList::UP_TO_PHP_82
89+
LevelSetList::UP_TO_PHP_82,
7690
])
7791
->withRules([
7892
DataProviderAnnotationToAttributeRector::class,
@@ -162,6 +176,21 @@
162176
new NewObjectToFunction(PipelineExtractor::class, 'from_pipeline'),
163177
new NewObjectToFunction(DataFrameExtractor::class, 'from_data_frame'),
164178

179+
// Adapters
180+
new NewObjectToFunction(AvroExtractor::class, 'Flow\ETL\DSL\Adapter\Avro\from_avro'),
181+
new NewObjectToFunction(CSVExtractor::class, 'Flow\ETL\Adapter\CSV\from_csv'),
182+
new NewObjectToFunction(DbalQueryExtractor::class, 'Flow\ETL\Adapter\Doctrine\from_dbal_query'),
183+
new NewObjectToFunction(ElasticsearchExtractor::class, 'Flow\ETL\Adapter\Elasticsearch\from_es'),
184+
new NewObjectToFunction(ExcelExtractor::class, 'Flow\ETL\Adapter\Excel\from_excel'),
185+
new NewObjectToFunction(GoogleSheetExtractor::class, 'Flow\ETL\Adapter\GoogleSheet\from_google_sheet'),
186+
new NewObjectToFunction(PsrHttpClientDynamicExtractor::class, 'Flow\ETL\Adapter\Http\from_dynamic_http_requests'),
187+
new NewObjectToFunction(PsrHttpClientStaticExtractor::class, 'Flow\ETL\Adapter\Http\from_static_http_requests'),
188+
new NewObjectToFunction(JsonExtractor::class, 'Flow\ETL\Adapter\JSON\from_json'),
189+
new NewObjectToFunction(JsonLinesExtractor::class, 'Flow\ETL\Adapter\JSON\from_json_lines'),
190+
new NewObjectToFunction(MeilisearchExtractor::class, 'Flow\ETL\Adapter\Meilisearch\from_meilisearch'),
191+
new NewObjectToFunction(ParquetExtractor::class, 'Flow\ETL\Adapter\Parquet\from_parquet'),
192+
new NewObjectToFunction(TextExtractor::class, 'Flow\ETL\Adapter\Text\from_text'),
193+
new NewObjectToFunction(XMLParserExtractor::class, 'Flow\ETL\Adapter\XML\from_xml'),
165194
]
166195
)
167196
->withSkip([
@@ -177,4 +206,4 @@
177206
importDocBlockNames: true,
178207
importShortClasses: false,
179208
removeUnusedImports: true
180-
);
209+
);

‎src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,31 +4,19 @@
44

55
namespace Flow\ETL\DSL\Adapter\Avro;
66

7-
use function Flow\ETL\DSL\from_all;
87
use Flow\ETL\Adapter\Avro\FlixTech\{AvroExtractor, AvroLoader};
9-
use Flow\ETL\{Extractor, Schema};
8+
use Flow\ETL\{Attribute\DocumentationDSL, Attribute\Module, Attribute\Type, Schema};
109
use Flow\Filesystem\Path;
1110

12-
function from_avro(Path|string|array $path) : Extractor
11+
#[DocumentationDSL(module: Module::AVRO, type: Type::EXTRACTOR)]
12+
function from_avro(Path|string $path) : AvroExtractor
1313
{
14-
if (\is_array($path)) {
15-
/** @var array<Extractor> $extractors */
16-
$extractors = [];
17-
18-
foreach ($path as $next_path) {
19-
$extractors[] = new AvroExtractor(
20-
\is_string($next_path) ? Path::realpath($next_path) : $next_path,
21-
);
22-
}
23-
24-
return from_all(...$extractors);
25-
}
26-
2714
return new AvroExtractor(
2815
\is_string($path) ? Path::realpath($path) : $path
2916
);
3017
}
3118

19+
#[DocumentationDSL(module: Module::AVRO, type: Type::LOADER)]
3220
function to_avro(Path|string $path, ?Schema $schema = null) : AvroLoader
3321
{
3422
return new AvroLoader(

‎src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Integration/AvroTest.php‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@
44

55
namespace Flow\ETL\Adapter\Avro\Tests\Integration;
66

7-
use function Flow\ETL\DSL\Adapter\Avro\to_avro;
7+
use function Flow\ETL\DSL\Adapter\Avro\{from_avro, to_avro};
88
use function Flow\ETL\DSL\{config, flow_context};
9-
use Flow\ETL\Adapter\Avro\FlixTech\AvroExtractor;
109
use Flow\ETL\Extractor\Signal;
1110
use Flow\ETL\Tests\FlowTestCase;
1211
use Flow\Filesystem\Path;
@@ -20,7 +19,7 @@ protected function setUp() : void
2019

2120
public function test_limit() : void
2221
{
23-
$extractor = new AvroExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.avro'));
22+
$extractor = from_avro(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.avro'));
2423
$extractor->changeLimit(2);
2524

2625
self::assertCount(
@@ -31,7 +30,7 @@ public function test_limit() : void
3130

3231
public function test_signal_stop() : void
3332
{
34-
$extractor = new AvroExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.avro'));
33+
$extractor = from_avro(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.avro'));
3534

3635
$generator = $extractor->extract(flow_context(config()));
3736

‎src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
use function Flow\ETL\Adapter\CSV\from_csv;
88
use function Flow\ETL\DSL\{df, print_schema, ref};
99
use function Flow\ETL\DSL\flow_context;
10-
use Flow\ETL\Adapter\CSV\CSVExtractor;
1110
use Flow\ETL\{Config, Row, Rows, Tests\FlowTestCase};
1211
use Flow\ETL\Extractor\Signal;
1312
use Flow\Filesystem\Path;
@@ -444,7 +443,7 @@ public function test_extracting_csv_with_multiline_strings() : void
444443
public function test_limit() : void
445444
{
446445

447-
$extractor = new CSVExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv'));
446+
$extractor = from_csv(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv'));
448447
$extractor->changeLimit(2);
449448

450449
self::assertCount(
@@ -477,7 +476,7 @@ public function test_loading_data_from_all_partitions() : void
477476

478477
public function test_signal_stop() : void
479478
{
480-
$extractor = new CSVExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv'));
479+
$extractor = from_csv(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv'));
481480

482481
$generator = $extractor->extract(flow_context(\Flow\ETL\DSL\config()));
483482

‎src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/DSL/functions.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ function from_excel(
2121
return new ExcelExtractor(\is_string($path) ? Path::realpath($path) : $path);
2222
}
2323

24+
#[DocumentationDSL(module: Module::EXCEL, type: DSLType::HELPER)]
2425
function is_valid_excel_sheet_name(string|ScalarFunction $sheet_name) : IsValidExcelSheetName
2526
{
2627
return new IsValidExcelSheetName($sheet_name);

‎src/adapter/etl-adapter-http/composer.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@
2929
"Flow\\": [
3030
"src/Flow"
3131
]
32-
}
32+
},
33+
"files": [
34+
"src/Flow/ETL/Adapter/Http/DSL/functions.php"
35+
]
3336
},
3437
"autoload-dev": {
3538
"psr-4": {

0 commit comments

Comments
 (0)