From faa0d61aae9a9889c00a37ef932fc46135934748 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Tue, 1 Jul 2025 15:22:56 -0700 Subject: [PATCH 1/3] feat: use_parent_params configurable --- .../declarative_component_schema.yaml | 5 + .../manifest_declarative_source.py | 6 +- .../models/declarative_component_schema.py | 7 + .../test_manifest_component_transformer.py | 217 ++++++++++++++++++ 4 files changed, 234 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 603a88e03..b4dfbb679 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4235,6 +4235,11 @@ definitions: - "$ref": "#/definitions/HttpComponentsResolver" - "$ref": "#/definitions/ConfigComponentsResolver" - "$ref": "#/definitions/ParametrizedComponentsResolver" + use_parent_parameters: + title: Use Parent Parameters + description: Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility. + type: boolean + default: true required: - type - stream_template diff --git a/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte_cdk/sources/declarative/manifest_declarative_source.py index 73abbe5d7..7452d5c68 100644 --- a/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -553,9 +553,13 @@ def _dynamic_stream_configs( for dynamic_stream in components_resolver.resolve_components( stream_template_config=stream_template_config ): + # Get the use_parent_parameters configuration from the dynamic definition + # Default to True for backward compatibility, since connectors were already using it by default when this param was added + use_parent_parameters = dynamic_definition.get("use_parent_parameters", True) + dynamic_stream = { **ManifestComponentTransformer().propagate_types_and_parameters( - "", dynamic_stream, {}, use_parent_parameters=True + "", dynamic_stream, {}, use_parent_parameters=use_parent_parameters ) } diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 1644071a4..e82c61ccc 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. + # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -2958,6 +2960,11 @@ class DynamicDeclarativeStream(BaseModel): description="Component resolve and populates stream templates with components values.", title="Components Resolver", ) + use_parent_parameters: Optional[bool] = Field( + True, + description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", + title="Use Parent Parameters", + ) ComplexFieldType.update_forward_refs() diff --git a/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py b/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py index 2a2dedbc1..d7da330c1 100644 --- a/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +++ b/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py @@ -569,3 +569,220 @@ def test_propagate_property_chunking(): transformer = ManifestComponentTransformer() actual_component = transformer.propagate_types_and_parameters("", component, {}) assert actual_component == expected_component + + +def test_use_parent_parameters_configuration(): + """Test that use_parent_parameters configuration controls parameter precedence.""" + # Test with use_parent_parameters=True (parent parameters take precedence) + component_with_parent_priority = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "name": "component_priority", + "url_base": "https://coffee.example.io/v1/", + "http_method": "GET", + "primary_key": "id", + "$parameters": { + "name": "component_priority", + }, + }, + "$parameters": { + "name": "parent_priority", + }, + }, + } + + expected_with_parent_priority = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + "name": "parent_priority", # Parent parameter takes precedence + "requester": { + "type": "HttpRequester", + "name": "component_priority", # Explicit value is not overridden + "url_base": "https://coffee.example.io/v1/", + "http_method": "GET", + "primary_key": "id", + "$parameters": { + "name": "parent_priority", # Parent parameter propagated to nested component + }, + }, + "$parameters": { + "name": "parent_priority", + }, + }, + } + + transformer = ManifestComponentTransformer() + actual_with_parent_priority = transformer.propagate_types_and_parameters( + "", component_with_parent_priority, {}, use_parent_parameters=True + ) + assert actual_with_parent_priority == expected_with_parent_priority + + # Test with use_parent_parameters=False (component parameters take precedence) + expected_with_component_priority = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + "name": "parent_priority", # Parent parameter takes precedence at this level + "requester": { + "type": "HttpRequester", + "name": "component_priority", # Component parameter takes precedence + "url_base": "https://coffee.example.io/v1/", + "http_method": "GET", + "primary_key": "id", + "$parameters": { + "name": "component_priority", + }, + }, + "$parameters": { + "name": "parent_priority", + }, + }, + } + + actual_with_component_priority = transformer.propagate_types_and_parameters( + "", component_with_parent_priority, {}, use_parent_parameters=False + ) + assert actual_with_component_priority == expected_with_component_priority + + +def test_use_parent_parameters_none_behavior(): + """Test that use_parent_parameters=None maintains backward compatibility.""" + component = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "name": "component_priority", + "url_base": "https://coffee.example.io/v1/", + "http_method": "GET", + "primary_key": "id", + "$parameters": { + "name": "component_priority", + }, + }, + "$parameters": { + "name": "parent_priority", + }, + }, + } + + expected_component_priority = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + "name": "parent_priority", # Parent parameter takes precedence (default behavior) + "requester": { + "type": "HttpRequester", + "name": "component_priority", # Component parameter takes precedence + "url_base": "https://coffee.example.io/v1/", + "http_method": "GET", + "primary_key": "id", + "$parameters": { + "name": "component_priority", + }, + }, + "$parameters": { + "name": "parent_priority", + }, + }, + } + + transformer = ManifestComponentTransformer() + actual = transformer.propagate_types_and_parameters( + "", component, {}, use_parent_parameters=None + ) + assert actual == expected_component_priority + + +def test_dynamic_stream_use_parent_parameters_configuration(): + """Test that use_parent_parameters configuration is properly read from dynamic stream definitions.""" + + transformer = ManifestComponentTransformer() + + # Only parent has $parameters + component = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + }, + "$parameters": {"name": "parent_name"}, + } + + # When use_parent_parameters=False, component parameters should take precedence (but there are none) + result_false = transformer.propagate_types_and_parameters( + "", component, {}, use_parent_parameters=False + ) + # When use_parent_parameters=True, parent parameters should take precedence (and are used) + result_true = transformer.propagate_types_and_parameters( + "", component, {}, use_parent_parameters=True + ) + + # In both cases, since only the parent has $parameters, the retriever should get "parent_name" + assert result_false["retriever"]["name"] == "parent_name" + assert result_true["retriever"]["name"] == "parent_name" + + # Now, add a $parameters to the retriever to see the difference + component_with_both = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + "$parameters": {"name": "retriever_name"}, + }, + "$parameters": {"name": "parent_name"}, + } + + result_false = transformer.propagate_types_and_parameters( + "", component_with_both, {}, use_parent_parameters=False + ) + result_true = transformer.propagate_types_and_parameters( + "", component_with_both, {}, use_parent_parameters=True + ) + + # When use_parent_parameters=False, retriever's own $parameters win + assert result_false["retriever"]["name"] == "retriever_name" + # When use_parent_parameters=True, parent's $parameters win + assert result_true["retriever"]["name"] == "parent_name" + + +def test_use_parent_parameters_simple(): + """Simple test to understand the use_parent_parameters behavior.""" + component = { + "type": "DeclarativeStream", + "retriever": { + "type": "SimpleRetriever", + "$parameters": { + "name": "component_name", + }, + }, + "$parameters": { + "name": "parent_name", + }, + } + + transformer = ManifestComponentTransformer() + + # Test with use_parent_parameters=True + result_true = transformer.propagate_types_and_parameters( + "", component, {}, use_parent_parameters=True + ) + print("use_parent_parameters=True:", result_true) + + # Test with use_parent_parameters=False + result_false = transformer.propagate_types_and_parameters( + "", component, {}, use_parent_parameters=False + ) + print("use_parent_parameters=False:", result_false) + + # Test with use_parent_parameters=None (default) + result_none = transformer.propagate_types_and_parameters( + "", component, {}, use_parent_parameters=None + ) + print("use_parent_parameters=None:", result_none) + + # For now, just assert that the results are different + assert result_true != result_false From f2f17f77266348562c0e43a44f2b2c3d519186b5 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Tue, 1 Jul 2025 15:30:55 -0700 Subject: [PATCH 2/3] chore: remove useless test --- .../test_manifest_component_transformer.py | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py b/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py index d7da330c1..4d1c9fb33 100644 --- a/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +++ b/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py @@ -747,42 +747,3 @@ def test_dynamic_stream_use_parent_parameters_configuration(): assert result_false["retriever"]["name"] == "retriever_name" # When use_parent_parameters=True, parent's $parameters win assert result_true["retriever"]["name"] == "parent_name" - - -def test_use_parent_parameters_simple(): - """Simple test to understand the use_parent_parameters behavior.""" - component = { - "type": "DeclarativeStream", - "retriever": { - "type": "SimpleRetriever", - "$parameters": { - "name": "component_name", - }, - }, - "$parameters": { - "name": "parent_name", - }, - } - - transformer = ManifestComponentTransformer() - - # Test with use_parent_parameters=True - result_true = transformer.propagate_types_and_parameters( - "", component, {}, use_parent_parameters=True - ) - print("use_parent_parameters=True:", result_true) - - # Test with use_parent_parameters=False - result_false = transformer.propagate_types_and_parameters( - "", component, {}, use_parent_parameters=False - ) - print("use_parent_parameters=False:", result_false) - - # Test with use_parent_parameters=None (default) - result_none = transformer.propagate_types_and_parameters( - "", component, {}, use_parent_parameters=None - ) - print("use_parent_parameters=None:", result_none) - - # For now, just assert that the results are different - assert result_true != result_false From 951053d2308d6e772ad2047b8f3d40d3646161d9 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Tue, 1 Jul 2025 15:38:27 -0700 Subject: [PATCH 3/3] chore: parametrize test --- .../test_manifest_component_transformer.py | 68 +++++++++---------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py b/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py index 4d1c9fb33..c741fa94a 100644 --- a/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +++ b/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py @@ -571,9 +571,32 @@ def test_propagate_property_chunking(): assert actual_component == expected_component -def test_use_parent_parameters_configuration(): +@pytest.mark.parametrize( + "use_parent_parameters, expected_retriever_name, expected_requester_name, expected_requester_params_name", + [ + pytest.param( + True, + "parent_priority", + "component_priority", + "parent_priority", + id="use_parent_parameters_true", + ), + pytest.param( + False, + "parent_priority", + "component_priority", + "component_priority", + id="use_parent_parameters_false", + ), + ], +) +def test_use_parent_parameters_configuration( + use_parent_parameters, + expected_retriever_name, + expected_requester_name, + expected_requester_params_name, +): """Test that use_parent_parameters configuration controls parameter precedence.""" - # Test with use_parent_parameters=True (parent parameters take precedence) component_with_parent_priority = { "type": "DeclarativeStream", "retriever": { @@ -594,19 +617,19 @@ def test_use_parent_parameters_configuration(): }, } - expected_with_parent_priority = { + expected_component = { "type": "DeclarativeStream", "retriever": { "type": "SimpleRetriever", - "name": "parent_priority", # Parent parameter takes precedence + "name": expected_retriever_name, "requester": { "type": "HttpRequester", - "name": "component_priority", # Explicit value is not overridden + "name": expected_requester_name, "url_base": "https://coffee.example.io/v1/", "http_method": "GET", "primary_key": "id", "$parameters": { - "name": "parent_priority", # Parent parameter propagated to nested component + "name": expected_requester_params_name, }, }, "$parameters": { @@ -616,37 +639,10 @@ def test_use_parent_parameters_configuration(): } transformer = ManifestComponentTransformer() - actual_with_parent_priority = transformer.propagate_types_and_parameters( - "", component_with_parent_priority, {}, use_parent_parameters=True + actual_component = transformer.propagate_types_and_parameters( + "", component_with_parent_priority, {}, use_parent_parameters=use_parent_parameters ) - assert actual_with_parent_priority == expected_with_parent_priority - - # Test with use_parent_parameters=False (component parameters take precedence) - expected_with_component_priority = { - "type": "DeclarativeStream", - "retriever": { - "type": "SimpleRetriever", - "name": "parent_priority", # Parent parameter takes precedence at this level - "requester": { - "type": "HttpRequester", - "name": "component_priority", # Component parameter takes precedence - "url_base": "https://coffee.example.io/v1/", - "http_method": "GET", - "primary_key": "id", - "$parameters": { - "name": "component_priority", - }, - }, - "$parameters": { - "name": "parent_priority", - }, - }, - } - - actual_with_component_priority = transformer.propagate_types_and_parameters( - "", component_with_parent_priority, {}, use_parent_parameters=False - ) - assert actual_with_component_priority == expected_with_component_priority + assert actual_component == expected_component def test_use_parent_parameters_none_behavior():