attempt to fix the Connector Builder tests

Author: Oleksandr Bazarnov

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,7 +4,7 @@
 
 
 from dataclasses import asdict, dataclass, field
-from typing import Any, Dict, List, Mapping
+from typing import Any, Dict, List, Mapping, Optional
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -54,12 +54,17 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
     return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
 
-def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
+def create_source(
+    config: Mapping[str, Any],
+    limits: TestLimits,
+    post_resolve_manifest: Optional[bool] = False,
+) -> ManifestDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
     return ManifestDeclarativeSource(
         config=config,
         emit_connector_builder_messages=True,
         source_config=manifest,
+        post_resolve_manifest=post_resolve_manifest,
         component_factory=ModelToComponentFactory(
             emit_connector_builder_messages=True,
             limit_pages_fetched_per_slice=limits.max_pages_per_slice,

airbyte_cdk/connector_builder/main.py

@@ -88,10 +88,10 @@ def handle_connector_builder_request(
         raise ValueError(f"Unrecognized command {command}.")
 
 
-def handle_request(args: List[str]) -> str:
+def handle_request(args: List[str], post_resolve_manifest: Optional[bool] = False) -> str:
     command, config, catalog, state = get_config_and_catalog_from_args(args)
     limits = get_limits(config)
-    source = create_source(config, limits)
+    source = create_source(config, limits, post_resolve_manifest=post_resolve_manifest)
     return orjson.dumps(
         AirbyteMessageSerializer.dump(
             handle_connector_builder_request(source, command, config, catalog, state, limits)

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -91,6 +91,7 @@ def __init__(
         debug: bool = False,
         emit_connector_builder_messages: bool = False,
         component_factory: Optional[ModelToComponentFactory] = None,
+        post_resolve_manifest: Optional[bool] = False,
     ) -> None:
         """
         Args:
@@ -129,6 +130,13 @@ def __init__(
                 self._declarative_component_schema,
             ).normalize()
 
+            # The manifest is now in a format that the Connector Builder UI can use.
+            # however, the local tests may depend on the completely resolved manifest.
+            if post_resolve_manifest:
+                propagated_source_config = ManifestReferenceResolver().preprocess_manifest(
+                    propagated_source_config
+                )
+
         self._source_config = propagated_source_config
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages

airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -4,7 +4,7 @@
 
 import copy
 import typing
-from typing import Any, Mapping, Optional
+from typing import Any, Dict, Mapping, Optional
 
 PARAMETERS_STR = "$parameters"
 
@@ -95,7 +95,7 @@ def propagate_types_and_parameters(
         declarative_component: Mapping[str, Any],
         parent_parameters: Mapping[str, Any],
         use_parent_parameters: Optional[bool] = None,
-    ) -> Mapping[str, Any]:
+    ) -> Dict[str, Any]:
         """
         Recursively transforms the specified declarative component and subcomponents to propagate parameters and insert the
         default component type if it was not already present. The resulting transformed components are a deep copy of the input

unit_tests/connector_builder/test_connector_builder_handler.py

@@ -503,7 +503,8 @@ def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catal
                 str(valid_resolve_manifest_config_file),
                 "--catalog",
                 str(dummy_catalog),
-            ]
+            ],
+            post_resolve_manifest=True,
         )
         assert patched_handle.call_count == 1
 
@@ -515,7 +516,14 @@ def test_handle_test_read(valid_read_config_file, configured_catalog):
         return_value=AirbyteMessage(type=MessageType.RECORD),
     ) as patch:
         handle_request(
-            ["read", "--config", str(valid_read_config_file), "--catalog", str(configured_catalog)]
+            [
+                "read",
+                "--config",
+                str(valid_read_config_file),
+                "--catalog",
+                str(configured_catalog),
+            ],
+            post_resolve_manifest=True,
         )
         assert patch.call_count == 1
 
@@ -867,7 +875,7 @@ def test_handle_429_response():
 
     config = TEST_READ_CONFIG
     limits = TestLimits()
-    source = create_source(config, limits)
+    source = create_source(config, limits, post_resolve_manifest=True)
 
     with patch("requests.Session.send", return_value=response) as mock_send:
         response = handle_connector_builder_request(
@@ -919,7 +927,14 @@ def test_missing_config(valid_resolve_manifest_config_file):
 def test_invalid_config_command(invalid_config_file, dummy_catalog):
     with pytest.raises(ValueError):
         handle_request(
-            ["read", "--config", str(invalid_config_file), "--catalog", str(dummy_catalog)]
+            [
+                "read",
+                "--config",
+                str(invalid_config_file),
+                "--catalog",
+                str(dummy_catalog),
+            ],
+            post_resolve_manifest=True,
         )
 
 
@@ -987,7 +1002,7 @@ def test_create_source():
 
     config = {"__injected_declarative_manifest": MANIFEST}
 
-    source = create_source(config, limits)
+    source = create_source(config, limits, post_resolve_manifest=True)
 
     assert isinstance(source, ManifestDeclarativeSource)
     assert source._constructor._limit_pages_fetched_per_slice == limits.max_pages_per_slice
@@ -1081,7 +1096,7 @@ def test_read_source(mock_http_stream):
 
     config = {"__injected_declarative_manifest": MANIFEST}
 
-    source = create_source(config, limits)
+    source = create_source(config, limits, post_resolve_manifest=True)
 
     output_data = read_stream(source, config, catalog, _A_PER_PARTITION_STATE, limits).record.data
     slices = output_data["slices"]
@@ -1128,7 +1143,7 @@ def test_read_source_single_page_single_slice(mock_http_stream):
 
     config = {"__injected_declarative_manifest": MANIFEST}
 
-    source = create_source(config, limits)
+    source = create_source(config, limits, post_resolve_manifest=True)
 
     output_data = read_stream(source, config, catalog, _A_PER_PARTITION_STATE, limits).record.data
     slices = output_data["slices"]
@@ -1214,7 +1229,7 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error
     test_manifest["streams"][0]["$parameters"]["url_base"] = url_base
     config = {"__injected_declarative_manifest": test_manifest}
 
-    source = create_source(config, limits)
+    source = create_source(config, limits, post_resolve_manifest=True)
 
     with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False):
         output_data = read_stream(
@@ -1310,7 +1325,7 @@ def test_handle_read_external_oauth_request(deployment_mode, token_url, expected
     )
     config = {"__injected_declarative_manifest": test_manifest}
 
-    source = create_source(config, limits)
+    source = create_source(config, limits, post_resolve_manifest=True)
 
     with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False):
         output_data = read_stream(