Skip to content
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
eeca9ef
Unify how we generate an array from Extractors
Bellangelo Dec 28, 2024
7c504cb
Use the new test case
Bellangelo Dec 28, 2024
c6a99b0
Clean-up files
Bellangelo Dec 28, 2024
5994b71
Add methods for modifying and asserting rows
Bellangelo Dec 28, 2024
db42e7c
Use the new methods
Bellangelo Dec 28, 2024
f7427e5
Add method for multi-row counting
Bellangelo Dec 28, 2024
5283eef
Use the methods in the rest of the files
Bellangelo Dec 28, 2024
8bf0e64
Import function
Bellangelo Dec 28, 2024
90114ad
Leave this class pure to test the implementation
Bellangelo Dec 28, 2024
61b96e2
Import of build-in function is forbidden
Bellangelo Dec 28, 2024
1581271
Merge branch '1.x' into extractor-test-helper
norberttech Dec 28, 2024
cd13e23
Convert helper function to an assertion
Bellangelo Dec 28, 2024
cb03235
Remove unused method
Bellangelo Dec 28, 2024
c33bc40
Convert helper method to an assertion
Bellangelo Dec 28, 2024
4b9b104
Fix code style
Bellangelo Dec 28, 2024
71b93d9
Follow the naming conventions of PHPUnit
Bellangelo Dec 28, 2024
fd444a1
Use a more descriptive name
Bellangelo Dec 28, 2024
050963e
Split logic into 2 methods
Bellangelo Dec 28, 2024
43a88c8
Rename test case and fix code style issues
Bellangelo Dec 28, 2024
a562ffb
Revert "Rename test case and fix code style issues"
Bellangelo Dec 28, 2024
9e9e195
Rename test case
Bellangelo Dec 28, 2024
802411b
Fix code style
Bellangelo Dec 28, 2024
87f5a00
Improve assertion naming
Bellangelo Dec 29, 2024
315d9f9
Assert by flatten first the rows
Bellangelo Dec 29, 2024
edef63a
Use the same implementation
Bellangelo Dec 29, 2024
7294309
Allow to override default flow context
Bellangelo Dec 29, 2024
9b69725
Allow to override default error message
Bellangelo Dec 29, 2024
466151b
Make methods static and final as PHPUnit does
Bellangelo Dec 29, 2024
1e1c2fe
Move class into the ETL root
Bellangelo Dec 29, 2024
6b300a0
Integration should extend the base FlowTestCase
Bellangelo Dec 29, 2024
cea9939
Use DSL functions instead of objects
Bellangelo Dec 29, 2024
2460322
Use self instead of directly accessing the FlowTestCase
Bellangelo Dec 29, 2024
38151c7
Merge branch '1.x' into extractor-test-helper
norberttech Dec 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@
namespace Flow\ETL\Tests\Unit\Extractor;

use function Flow\ETL\DSL\{from_all, int_entry};
use Flow\ETL\{Config, Extractor, FlowContext, Row, Rows};
use PHPUnit\Framework\TestCase;
use Flow\ETL\{Extractor, FlowContext, Row, Rows};

final class ChainExtractorTest extends TestCase
final class ChainExtractorTest extends FlowTestCase
{
public function test_chain_extractor() : void
{
Expand All @@ -29,14 +28,14 @@ public function extract(FlowContext $context) : \Generator
},
);

self::assertEquals(
$this->assertExtractorEqualsRows(
[
new Rows(Row::create(int_entry('id', 1))),
new Rows(Row::create(int_entry('id', 2))),
new Rows(Row::create(int_entry('id', 3))),
new Rows(Row::create(int_entry('id', 4))),
],
\iterator_to_array($extractor->extract(new FlowContext(Config::default())))
$extractor
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,26 @@

use Flow\ETL\Extractor\ChunkExtractor;
use Flow\ETL\Tests\Double\FakeExtractor;
use Flow\ETL\{Config, FlowContext};
use PHPUnit\Framework\TestCase;

final class ChunkExtractorTest extends TestCase
final class ChunkExtractorTest extends FlowTestCase
{
public function test_chunk_extractor() : void
{
$extractor = new ChunkExtractor(new FakeExtractor($batches = 100), $chunkSize = 10);

self::assertCount(
$this->assertExtractorCountBatches(
$batches / $chunkSize,
\iterator_to_array($extractor->extract(new FlowContext(Config::default())))
$extractor
);
}

public function test_chunk_extractor_with_chunk_size_greater_than_() : void
{
$extractor = new ChunkExtractor(new FakeExtractor(total: 20), chunkSize: 25);

self::assertCount(
$this->assertExtractorCountBatches(
1,
\iterator_to_array($extractor->extract(new FlowContext(Config::default())))
$extractor
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,25 @@
namespace Flow\ETL\Tests\Unit\Extractor;

use function Flow\ETL\DSL\{df, from_data_frame, from_rows, row, rows, str_entry};
use Flow\ETL\{Config, FlowContext};
use PHPUnit\Framework\TestCase;

final class DataFrameExtractorTest extends TestCase
final class DataFrameExtractorTest extends FlowTestCase
{
public function test_extracting_from_another_data_frame() : void
{
self::assertEquals(
$extractor = from_data_frame(
df()->read(from_rows(
rows(
row(str_entry('value', 'test')),
row(str_entry('value', 'test')),
),
rows(
row(str_entry('value', 'test')),
row(str_entry('value', 'test')),
)
))
);

$this->assertExtractorEqualsRows(
[
rows(
row(str_entry('value', 'test')),
Expand All @@ -23,20 +34,7 @@ public function test_extracting_from_another_data_frame() : void
row(str_entry('value', 'test')),
),
],
\iterator_to_array(
from_data_frame(
df()->read(from_rows(
rows(
row(str_entry('value', 'test')),
row(str_entry('value', 'test')),
),
rows(
row(str_entry('value', 'test')),
row(str_entry('value', 'test')),
)
)),
)->extract(new FlowContext(Config::default()))
),
$extractor
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,15 @@

use function Flow\ETL\DSL\{files, flow_context};
use Flow\ETL\Extractor\Signal;
use PHPUnit\Framework\TestCase;

final class FilesExtractorTest extends TestCase
final class FilesExtractorTest extends FlowTestCase
{
public function test_extracting_files_from_directory() : void
{
$extractor = files(__DIR__ . '/Fixtures/FileListExtractor/*');

$totalRows = 0;

foreach ($extractor->extract(flow_context()) as $rows) {
self::assertCount(1, $rows);
$totalRows += $rows->count();
}

self::assertEquals(3, $totalRows);
$this->assertExtractorCountRows(3, $extractor);
$this->assertExtractorCountRowsPerBatch(1, $extractor);
}

public function test_extracting_files_from_directory_after_getting_stop_signal() : void
Expand All @@ -43,28 +36,16 @@ public function test_extracting_files_from_directory_recursive() : void
{
$extractor = files(__DIR__ . '/Fixtures/FileListExtractor/**/*');

$totalRows = 0;

foreach ($extractor->extract(flow_context()) as $rows) {
self::assertCount(1, $rows);
$totalRows += $rows->count();
}

self::assertEquals(6, $totalRows);
$this->assertExtractorCountRows(6, $extractor);
$this->assertExtractorCountRowsPerBatch(1, $extractor);
}

public function test_extracting_files_from_directory_with_limit() : void
{
$extractor = files(__DIR__ . '/Fixtures/FileListExtractor/**/*');
$extractor->changeLimit(2);

$totalRows = 0;

foreach ($extractor->extract(flow_context()) as $rows) {
self::assertCount(1, $rows);
$totalRows += $rows->count();
}

self::assertEquals(2, $totalRows);
$this->assertExtractorCountRows(2, $extractor);
$this->assertExtractorCountRowsPerBatch(1, $extractor);
}
}
71 changes: 71 additions & 0 deletions src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FlowTestCase.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
<?php

declare(strict_types=1);

namespace Flow\ETL\Tests\Unit\Extractor;

use Flow\ETL\{Config, Extractor, FlowContext, Rows};
use PHPUnit\Framework\TestCase;

abstract class FlowTestCase extends TestCase
{
public function assertExtractorCountBatches(int $expectedCount, Extractor $extractor) : void
{
static::assertCount(
$expectedCount,
\iterator_to_array($extractor->extract(new FlowContext(Config::default())))
);
}

public function assertExtractorCountRows(int $expectedCount, Extractor $extractor) : void
{
$totalRows = 0;

foreach ($extractor->extract(new FlowContext(Config::default())) as $rows) {
$totalRows += $rows->count();
}

static::assertSame($expectedCount, $totalRows);
}

public function assertExtractorCountRowsPerBatch(int $expectedCount, Extractor $extractor) : void
{
$extractorContainsBatches = false;

foreach ($extractor->extract(new FlowContext(Config::default())) as $rows) {
static::assertCount($expectedCount, $rows);
$extractorContainsBatches = true;
}

if (!$extractorContainsBatches) {
static::fail('Extractor does not contain any batches');
}
}

/**
* @param array<Rows> $expectedRows
* @param Extractor $extractor
*/
public function assertExtractorEqualsRows(array $expectedRows, Extractor $extractor) : void
{
static::assertEquals(
$expectedRows,
\iterator_to_array($extractor->extract(new FlowContext(Config::default())))
);
}

/**
* @param array<mixed> $expectedArray
* @param Extractor $extractor
*/
public function assertExtractorSameArray(array $expectedArray, Extractor $extractor) : void
{
$data = [];

foreach ($extractor->extract(new FlowContext(Config::default())) as $rowsData) {
$data = [...$data, ...$rowsData->toArray()];
}

static::assertSame($expectedArray, $data);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@
use function Flow\ETL\DSL\{from_memory, int_entry, str_entry, to_memory};
use Flow\ETL\Memory\ArrayMemory;
use Flow\ETL\{Config, FlowContext, Row, Rows};
use PHPUnit\Framework\TestCase;

final class MemoryExtractorTest extends TestCase
final class MemoryExtractorTest extends FlowTestCase
{
public function test_memory_extractor() : void
{
Expand All @@ -27,21 +26,15 @@ public function test_memory_extractor() : void

$extractor = from_memory($memory);

$data = [];

foreach ($extractor->extract(new FlowContext(Config::default())) as $rowsData) {
$data = [...$data, ...$rowsData->toArray()];
}

self::assertSame(
$this->assertExtractorSameArray(
[
['number' => 1, 'name' => 'one'],
['number' => 2, 'name' => 'two'],
['number' => 3, 'name' => 'tree'],
['number' => 4, 'name' => 'four'],
['number' => 5, 'name' => 'five'],
],
$data
$extractor
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@
use function Flow\ETL\DSL\int_entry;
use Flow\ETL\Extractor\{PipelineExtractor, RowsExtractor};
use Flow\ETL\Pipeline\SynchronousPipeline;
use Flow\ETL\{Config, FlowContext, Row, Rows};
use PHPUnit\Framework\TestCase;
use Flow\ETL\{Row, Rows};

final class PipelineExtractorTest extends TestCase
final class PipelineExtractorTest extends FlowTestCase
{
public function test_pipeline_extractor() : void
{
Expand All @@ -22,9 +21,6 @@ public function test_pipeline_extractor() : void

$extractor = new PipelineExtractor($pipeline);

self::assertCount(
3,
\iterator_to_array($extractor->extract(new FlowContext(Config::default())))
);
$this->assertExtractorCountBatches(3, $extractor);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@
namespace Flow\ETL\Tests\Unit\Extractor;

use function Flow\ETL\DSL\{from_rows, int_entry, str_entry};
use Flow\ETL\{Config, FlowContext, Row, Rows};
use PHPUnit\Framework\TestCase;
use Flow\ETL\{Row, Rows};

final class RowsExtractorTest extends TestCase
final class RowsExtractorTest extends FlowTestCase
{
public function test_process_extractor() : void
{
Expand All @@ -22,21 +21,15 @@ public function test_process_extractor() : void

$extractor = from_rows($rows);

$data = [];

foreach ($extractor->extract(new FlowContext(Config::default())) as $rowsData) {
$data = [...$data, ...$rowsData->toArray()];
}

self::assertSame(
$this->assertExtractorSameArray(
[
['number' => 1, 'name' => 'one'],
['number' => 2, 'name' => 'two'],
['number' => 3, 'name' => 'tree'],
['number' => 4, 'name' => 'four'],
['number' => 5, 'name' => 'five'],
],
$rows->toArray()
$extractor
);
}
}
Loading
Loading