Skip to content

Commit 4d8c918

Browse files
committed
Add unit test for model_to_component_factory.py
1 parent cf782a2 commit 4d8c918

File tree

3 files changed

+65
-5
lines changed

3 files changed

+65
-5
lines changed

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,7 @@ def _init_mappings(self) -> None:
628628
UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
629629
RateModel: self.create_rate,
630630
HttpRequestRegexMatcherModel: self.create_http_request_matcher,
631+
GroupingPartitionRouterModel: self.create_grouping_partition_router,
631632
}
632633

633634
# Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -3065,6 +3066,5 @@ def create_grouping_partition_router(
30653066
group_size=model.group_size,
30663067
underlying_partition_router=underlying_router,
30673068
deduplicate=model.deduplicate if model.deduplicate is not None else True,
3068-
parameters=model.parameters or {},
30693069
config=config,
30703070
)

unit_tests/sources/declarative/parsers/test_model_to_component_factory.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@
6565
from airbyte_cdk.sources.declarative.models import DatetimeBasedCursor as DatetimeBasedCursorModel
6666
from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel
6767
from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel
68+
from airbyte_cdk.sources.declarative.models import (
69+
GroupingPartitionRouter as GroupingPartitionRouterModel,
70+
)
6871
from airbyte_cdk.sources.declarative.models import HttpRequester as HttpRequesterModel
6972
from airbyte_cdk.sources.declarative.models import JwtAuthenticator as JwtAuthenticatorModel
7073
from airbyte_cdk.sources.declarative.models import ListPartitionRouter as ListPartitionRouterModel
@@ -96,6 +99,7 @@
9699
from airbyte_cdk.sources.declarative.partition_routers import (
97100
AsyncJobPartitionRouter,
98101
CartesianProductStreamSlicer,
102+
GroupingPartitionRouter,
99103
ListPartitionRouter,
100104
SinglePartitionRouter,
101105
SubstreamPartitionRouter,
@@ -3840,3 +3844,59 @@ def test_api_budget_fixed_window_policy():
38403844
assert matcher._method == "GET"
38413845
assert matcher._url_base == "https://example.org"
38423846
assert matcher._url_path_pattern.pattern == "/v2/data"
3847+
3848+
3849+
def test_create_grouping_partition_router_with_underlying_router():
3850+
content = """
3851+
schema_loader:
3852+
file_path: "./source_example/schemas/{{ parameters['name'] }}.yaml"
3853+
name: "{{ parameters['stream_name'] }}"
3854+
retriever:
3855+
requester:
3856+
type: "HttpRequester"
3857+
path: "example"
3858+
record_selector:
3859+
extractor:
3860+
field_path: []
3861+
stream_A:
3862+
type: DeclarativeStream
3863+
name: "A"
3864+
primary_key: "id"
3865+
$parameters:
3866+
retriever: "#/retriever"
3867+
url_base: "https://airbyte.io"
3868+
schema_loader: "#/schema_loader"
3869+
sub_partition_router:
3870+
type: SubstreamPartitionRouter
3871+
parent_stream_configs:
3872+
- stream: "#/stream_A"
3873+
parent_key: id
3874+
partition_field: repository_id
3875+
partition_router:
3876+
type: GroupingPartitionRouter
3877+
underlying_partition_router: "#/sub_partition_router"
3878+
group_size: 2
3879+
"""
3880+
parsed_manifest = YamlDeclarativeSource._parse(content)
3881+
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
3882+
partition_router_manifest = transformer.propagate_types_and_parameters(
3883+
"", resolved_manifest["partition_router"], {}
3884+
)
3885+
3886+
partition_router = factory.create_component(
3887+
model_type=GroupingPartitionRouterModel,
3888+
component_definition=partition_router_manifest,
3889+
config=input_config,
3890+
)
3891+
3892+
# Test the created partition router
3893+
assert isinstance(partition_router, GroupingPartitionRouter)
3894+
assert isinstance(partition_router.underlying_partition_router, SubstreamPartitionRouter)
3895+
assert partition_router.group_size == 2
3896+
3897+
# Test the underlying partition router
3898+
parent_stream_configs = partition_router.underlying_partition_router.parent_stream_configs
3899+
assert len(parent_stream_configs) == 1
3900+
assert isinstance(parent_stream_configs[0].stream, DeclarativeStream)
3901+
assert parent_stream_configs[0].parent_key.eval({}) == "id"
3902+
assert parent_stream_configs[0].partition_field.eval({}) == "repository_id"

unit_tests/sources/declarative/partition_routers/test_grouping_partition_router.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@
66
from unittest.mock import MagicMock
77

88
import pytest
9-
from unit_tests.sources.declarative.partition_routers.test_substream_partition_router import (
10-
MockStream,
11-
parent_slices,
12-
) # Reuse MockStream and parent_slices
139

1410
from airbyte_cdk.sources.declarative.partition_routers import (
1511
GroupingPartitionRouter,
@@ -19,6 +15,10 @@
1915
ParentStreamConfig,
2016
)
2117
from airbyte_cdk.sources.types import StreamSlice
18+
from unit_tests.sources.declarative.partition_routers.test_substream_partition_router import (
19+
MockStream,
20+
parent_slices,
21+
) # Reuse MockStream and parent_slices
2222

2323

2424
@pytest.fixture

0 commit comments

Comments
 (0)