Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
eeca9ef
Unify how we generate an array from Extractors
Bellangelo Dec 28, 2024
7c504cb
Use the new test case
Bellangelo Dec 28, 2024
c6a99b0
Clean-up files
Bellangelo Dec 28, 2024
5994b71
Add methods for modifying and asserting rows
Bellangelo Dec 28, 2024
db42e7c
Use the new methods
Bellangelo Dec 28, 2024
f7427e5
Add method for multi-row counting
Bellangelo Dec 28, 2024
5283eef
Use the methods in the rest of the files
Bellangelo Dec 28, 2024
8bf0e64
Import function
Bellangelo Dec 28, 2024
90114ad
Leave this class pure to test the implementation
Bellangelo Dec 28, 2024
61b96e2
Import of build-in function is forbidden
Bellangelo Dec 28, 2024
1581271
Merge branch '1.x' into extractor-test-helper
norberttech Dec 28, 2024
cd13e23
Convert helper function to an assertion
Bellangelo Dec 28, 2024
cb03235
Remove unused method
Bellangelo Dec 28, 2024
c33bc40
Convert helper method to an assertion
Bellangelo Dec 28, 2024
4b9b104
Fix code style
Bellangelo Dec 28, 2024
71b93d9
Follow the naming conventions of PHPUnit
Bellangelo Dec 28, 2024
fd444a1
Use a more descriptive name
Bellangelo Dec 28, 2024
050963e
Split logic into 2 methods
Bellangelo Dec 28, 2024
43a88c8
Rename test case and fix code style issues
Bellangelo Dec 28, 2024
a562ffb
Revert "Rename test case and fix code style issues"
Bellangelo Dec 28, 2024
9e9e195
Rename test case
Bellangelo Dec 28, 2024
802411b
Fix code style
Bellangelo Dec 28, 2024
87f5a00
Improve assertion naming
Bellangelo Dec 29, 2024
315d9f9
Assert by flatten first the rows
Bellangelo Dec 29, 2024
edef63a
Use the same implementation
Bellangelo Dec 29, 2024
7294309
Allow to override default flow context
Bellangelo Dec 29, 2024
9b69725
Allow to override default error message
Bellangelo Dec 29, 2024
466151b
Make methods static and final as PHPUnit does
Bellangelo Dec 29, 2024
1e1c2fe
Move class into the ETL root
Bellangelo Dec 29, 2024
6b300a0
Integration should extend the base FlowTestCase
Bellangelo Dec 29, 2024
cea9939
Use DSL functions instead of objects
Bellangelo Dec 29, 2024
2460322
Use self instead of directly accessing the FlowTestCase
Bellangelo Dec 29, 2024
38151c7
Merge branch '1.x' into extractor-test-helper
norberttech Dec 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
use function Flow\ETL\Adapter\XML\{from_xml, to_xml};
use function Flow\ETL\DSL\{df, from_array, overwrite, ref};
use Flow\ETL\Tests\Double\FakeExtractor;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class XMLLoaderTest extends IntegrationTestCase
final class XMLLoaderTest extends FlowIntegrationTestCase
{
public function test_partitioning_xml_file() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
Flow,
FlowContext,
PHP\Type\Caster,
Tests\Integration\IntegrationTestCase};
Tests\Integration\FlowIntegrationTestCase};
use Flow\Filesystem\Path;

final class XMLParserExtractorTest extends IntegrationTestCase
final class XMLParserExtractorTest extends FlowIntegrationTestCase
{
public function test_limit() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
use function Flow\ETL\DSL\type_string;
use Flow\ETL\Adapter\XML\XMLReaderExtractor;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\{Config, Flow, FlowContext, PHP\Type\Caster, Tests\Integration\IntegrationTestCase};
use Flow\ETL\{Config, Flow, FlowContext, PHP\Type\Caster, Tests\Integration\FlowIntegrationTestCase};
use Flow\Filesystem\Path;

final class XMLReaderExtractorTest extends IntegrationTestCase
final class XMLReaderExtractorTest extends FlowIntegrationTestCase
{
public function test_limit() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
use function Flow\ETL\Adapter\XML\from_xml;
use function Flow\ETL\DSL\{datetime_schema, df, int_schema, ref, schema, type_int};
use function Flow\Filesystem\DSL\path;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class XMLTest extends IntegrationTestCase
final class XMLTest extends FlowIntegrationTestCase
{
public function test_transforming_xml_into_a_tabular_dataset() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@

use function Flow\Filesystem\Bridge\AsyncAWS\DSL\aws_s3_client;
use AsyncAws\S3\S3Client;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\Filesystem\Path;

abstract class AsyncAWSS3TestCase extends IntegrationTestCase
abstract class AsyncAWSS3TestCase extends FlowIntegrationTestCase
{
protected function setUp() : void
{
Expand Down
94 changes: 94 additions & 0 deletions src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
<?php

declare(strict_types=1);

namespace Flow\ETL\Tests;

use function Flow\ETL\DSL\{flow_context, rows};
use Flow\ETL\{Extractor, FlowContext, Rows};
use PHPUnit\Framework\TestCase;

abstract class FlowTestCase extends TestCase
{
final public static function assertExtractedBatchesCount(
int $expectedCount,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
) : void {
$flowContext = $flowContext ?? flow_context();

static::assertCount(
$expectedCount,
\iterator_to_array($extractor->extract($flowContext)),
$message
);
}

final public static function assertExtractedBatchesSize(
int $expectedCount,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
) : void {
$flowContext = $flowContext ?? flow_context();
$extractorContainsBatches = false;

foreach ($extractor->extract($flowContext) as $rows) {
static::assertCount($expectedCount, $rows, $message);
$extractorContainsBatches = true;
}

if (!$extractorContainsBatches) {
static::fail('Extractor does not contain any batches');
}
}

final public static function assertExtractedRowsAsArrayEquals(
array $expectedArray,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
) : void {
$flowContext = $flowContext ?? flow_context();
$extractedRows = rows();

foreach ($extractor->extract($flowContext) as $nextRows) {
$extractedRows = $extractedRows->merge($nextRows);
}

static::assertEquals($expectedArray, $extractedRows->toArray(), $message);
}

final public static function assertExtractedRowsCount(
int $expectedCount,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
) : void {
$flowContext = $flowContext ?? flow_context();
$totalRows = 0;

foreach ($extractor->extract($flowContext) as $rows) {
$totalRows += $rows->count();
}

static::assertSame($expectedCount, $totalRows, $message);
}

final public static function assertExtractedRowsEquals(
Rows $expectedRows,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
) : void {
$flowContext = $flowContext ?? flow_context();
$extractedRows = rows();

foreach ($extractor->extract($flowContext) as $nextRows) {
$extractedRows = $extractedRows->merge($nextRows);
}

static::assertEquals($expectedRows, $extractedRows, $message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
use function Flow\ETL\DSL\{row, rows, str_entry};
use Flow\ETL\Cache\{CacheIndex};
use Flow\ETL\Exception\KeyNotInCacheException;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

abstract class CacheBaseTestSuite extends IntegrationTestCase
abstract class CacheBaseTestSuite extends FlowIntegrationTestCase
{
protected function setUp() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

use function Flow\ETL\Adapter\Text\from_text;
use function Flow\ETL\DSL\{datetime_schema, df, float_schema, from_array, int_schema, schema, str_schema};
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class AnalyzeTest extends IntegrationTestCase
final class AnalyzeTest extends FlowIntegrationTestCase
{
public function test_analyzing_csv_file_with_auto_cast() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

use function Flow\ETL\DSL\{df, from_array, lit, ref, to_branch, to_memory};
use Flow\ETL\Memory\ArrayMemory;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class BranchingTest extends IntegrationTestCase
final class BranchingTest extends FlowIntegrationTestCase
{
public function test_branching() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Cache\Implementation\InMemoryCache;
use Flow\ETL\Tests\Double\FakeExtractor;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\ETL\{Extractor, FlowContext, Rows};

final class CacheTest extends IntegrationTestCase
final class CacheTest extends FlowIntegrationTestCase
{
public function test_cache() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
use function Flow\ETL\DSL\config_builder;
use Flow\ETL\Config\Cache\CacheConfig;
use Flow\ETL\Sort\SortAlgorithms;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class ConfigBuilderTest extends IntegrationTestCase
final class ConfigBuilderTest extends FlowIntegrationTestCase
{
public function test_creating_custom_cache_dir() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

use Flow\ETL\DataFrame;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class DataFrameJsonTest extends IntegrationTestCase
final class DataFrameJsonTest extends FlowIntegrationTestCase
{
public function test_building_data_frame_from_json() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ enum_entry,
type_string,
xml_entry};
use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\ETL\{Extractor, FlowContext, Rows};

final class DisplayTest extends IntegrationTestCase
final class DisplayTest extends FlowIntegrationTestCase
{
public function test_display() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
namespace Flow\ETL\Tests\Integration\DataFrame;

use function Flow\ETL\DSL\{df, from_array, ref};
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class FilterTest extends IntegrationTestCase
final class FilterTest extends FlowIntegrationTestCase
{
public function test_multiple_filters() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@
uuid_schema,
window};
use Flow\ETL\Memory\ArrayMemory;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\ETL\{Loader, Rows};

final class GroupByTest extends IntegrationTestCase
final class GroupByTest extends FlowIntegrationTestCase
{
public function test_group_by_array() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@

use function Flow\ETL\DSL\{datetime_entry, df, from_rows, int_entry, row, rows, str_entry};
use Flow\ETL\Join\Expression;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\ETL\{Flow, Join\Join, Loader};

final class JoinTest extends IntegrationTestCase
final class JoinTest extends FlowIntegrationTestCase
{
public function test_join_inner() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
type_structure};
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Row\Entry\{IntegerEntry};
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\ETL\{Extractor, FlowContext, Row, Rows};

final class LimitTest extends IntegrationTestCase
final class LimitTest extends FlowIntegrationTestCase
{
public function test_exceeding_the_limit_in_one_rows_set() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
type_list,
type_string};
use Flow\ETL\Row;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class MapTest extends IntegrationTestCase
final class MapTest extends FlowIntegrationTestCase
{
public function test_using_map_to_replace_nullable_lists() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@
rows_partitioned,
str_entry};
use function Flow\Filesystem\DSL\partition;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\ETL\{Rows};
use Flow\Filesystem\Partition;

final class PartitioningTest extends IntegrationTestCase
final class PartitioningTest extends FlowIntegrationTestCase
{
public function test_dropping_partitions() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
namespace Flow\ETL\Tests\Integration\DataFrame;

use function Flow\ETL\DSL\{bool_entry, df, from_rows, int_entry, json_entry, ref, str_entry};
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\ETL\Transformer\StyleConverter\StringStyles;
use Flow\ETL\{Row, Rows};

final class RenameTest extends IntegrationTestCase
final class RenameTest extends FlowIntegrationTestCase
{
public function test_rename() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
string_entry};
use Flow\ETL\Pipeline\SynchronousPipeline;
use Flow\ETL\Row\Schema;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class SchemaTest extends IntegrationTestCase
final class SchemaTest extends FlowIntegrationTestCase
{
public function test_extraction_according_to_schema() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
use Flow\ETL\Config;
use Flow\ETL\Monitoring\Memory\Unit;
use Flow\ETL\Tests\Double\{FakeExtractor};
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class SortTest extends IntegrationTestCase
final class SortTest extends FlowIntegrationTestCase
{
public function test_etl_sort_by_external_sort() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Cache\Implementation\InMemoryCache;
use Flow\ETL\Extractor\CacheExtractor;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

final class CacheExtractorTest extends IntegrationTestCase
final class CacheExtractorTest extends FlowIntegrationTestCase
{
public function test_extracting_rows_from_cache() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
namespace Flow\ETL\Tests\Integration\Extractor;

use function Flow\ETL\DSL\{flow_context, from_path_partitions, rows};
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
use Flow\Filesystem\Path;

final class PathPartitionsExtractorTest extends IntegrationTestCase
final class PathPartitionsExtractorTest extends FlowIntegrationTestCase
{
public function test_extracting_data_from_path_partitions() : void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams;

use Flow\ETL\Filesystem\FilesystemStreams;
use Flow\ETL\Tests\Integration\IntegrationTestCase;
use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;

abstract class FilesystemStreamsTestCase extends IntegrationTestCase
abstract class FilesystemStreamsTestCase extends FlowIntegrationTestCase
{
protected function filesDirectory() : string
{
Expand Down
Loading
Loading