removed post_resolve_manifest flag

Author: Oleksandr Bazarnov

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,7 +4,7 @@
 
 
 from dataclasses import asdict, dataclass, field
-from typing import Any, ClassVar, Dict, List, Mapping, Optional
+from typing import Any, ClassVar, Dict, List, Mapping
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -55,6 +55,7 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
     max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
     return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
+
 def normalize_manifest(config: Mapping[str, Any]) -> bool:
     """
     Check if the manifest should be normalized.
@@ -63,13 +64,6 @@ def normalize_manifest(config: Mapping[str, Any]) -> bool:
     """
     return config.get("__requires_normalization", False)
 
-def post_resolve_manifest(config: Mapping[str, Any]) -> bool:
-    """
-    Check if the manifest should be post-resolved.
-    :param config: The configuration to check
-    :return: True if the manifest should be post-resolved, False otherwise.
-    """
-    return config.get("__post_resolve_manifest", False)
 
 def create_source(
     config: Mapping[str, Any],
@@ -81,7 +75,6 @@ def create_source(
         emit_connector_builder_messages=True,
         source_config=manifest,
         normalize_manifest=normalize_manifest(config),
-        post_resolve_manifest=post_resolve_manifest(config),
         component_factory=ModelToComponentFactory(
             emit_connector_builder_messages=True,
             limit_pages_fetched_per_slice=limits.max_pages_per_slice,

airbyte_cdk/connector_builder/main.py

@@ -88,10 +88,10 @@ def handle_connector_builder_request(
         raise ValueError(f"Unrecognized command {command}.")
 
 
-def handle_request(args: List[str], post_resolve_manifest: Optional[bool] = False) -> str:
+def handle_request(args: List[str]) -> str:
     command, config, catalog, state = get_config_and_catalog_from_args(args)
     limits = get_limits(config)
-    source = create_source(config, limits, post_resolve_manifest=post_resolve_manifest)
+    source = create_source(config, limits)
     return orjson.dumps(
         AirbyteMessageSerializer.dump(
             handle_connector_builder_request(source, command, config, catalog, state, limits)

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -91,7 +91,7 @@ def __init__(
         debug: bool = False,
         emit_connector_builder_messages: bool = False,
         component_factory: Optional[ModelToComponentFactory] = None,
-        post_resolve_manifest: Optional[bool] = False,
+        normalize_manifest: Optional[bool] = False,
     ) -> None:
         """
         Args:
@@ -100,6 +100,8 @@ def __init__(
             debug: True if debug mode is enabled.
             emit_connector_builder_messages: True if messages should be emitted to the connector builder.
             component_factory: optional factory if ModelToComponentFactory's default behavior needs to be tweaked.
+            normalize_manifest: Optional flag to indicate if the manifest should be normalized.
+            post_resolve_manifest: Optional flag to indicate if the manifest should be resolved after normalization.
         """
         self.logger = logging.getLogger(f"airbyte.{self.name}")
 
@@ -119,24 +121,16 @@ def __init__(
             "", resolved_source_config, {}
         )
 
-        if emit_connector_builder_messages:
-            # Connector Builder Ui rendering requires the manifest to be in a specific format.
+        if normalize_manifest:
+            # Connector Builder UI rendering requires the manifest to be in a specific format.
             # 1) references have been resolved
-            # 2) deprecated fields have been migrated
-            # 3) the commonly used definitions are extracted to the `definitions.shared.*`
-            # 4) ! the normalized manifest could be validated after the additional UI post-processing.
+            # 2) the commonly used definitions are extracted to the `definitions.shared.*`
+            # 3) ! the normalized manifest could be validated only after the additional UI post-processing.
             propagated_source_config = ManifestNormalizer(
                 propagated_source_config,
                 self._declarative_component_schema,
             ).normalize()
 
-            # The manifest is now in a format that the Connector Builder UI can use.
-            # however, the local tests may depend on the completely resolved manifest.
-            if post_resolve_manifest:
-                propagated_source_config = ManifestReferenceResolver().preprocess_manifest(
-                    propagated_source_config
-                )
-
         self._source_config = propagated_source_config
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages

unit_tests/connector_builder/test_connector_builder_handler.py

@@ -504,7 +504,6 @@ def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catal
                 "--catalog",
                 str(dummy_catalog),
             ],
-            post_resolve_manifest=True,
         )
         assert patched_handle.call_count == 1
 
@@ -523,7 +522,6 @@ def test_handle_test_read(valid_read_config_file, configured_catalog):
                 "--catalog",
                 str(configured_catalog),
             ],
-            post_resolve_manifest=True,
         )
         assert patch.call_count == 1
 
@@ -875,7 +873,7 @@ def test_handle_429_response():
 
     config = TEST_READ_CONFIG
     limits = TestLimits()
-    source = create_source(config, limits, post_resolve_manifest=True)
+    source = create_source(config, limits)
 
     with patch("requests.Session.send", return_value=response) as mock_send:
         response = handle_connector_builder_request(
@@ -934,7 +932,6 @@ def test_invalid_config_command(invalid_config_file, dummy_catalog):
                 "--catalog",
                 str(dummy_catalog),
             ],
-            post_resolve_manifest=True,
         )
 
 
@@ -1002,7 +999,7 @@ def test_create_source():
 
     config = {"__injected_declarative_manifest": MANIFEST}
 
-    source = create_source(config, limits, post_resolve_manifest=True)
+    source = create_source(config, limits)
 
     assert isinstance(source, ManifestDeclarativeSource)
     assert source._constructor._limit_pages_fetched_per_slice == limits.max_pages_per_slice
@@ -1096,7 +1093,7 @@ def test_read_source(mock_http_stream):
 
     config = {"__injected_declarative_manifest": MANIFEST}
 
-    source = create_source(config, limits, post_resolve_manifest=True)
+    source = create_source(config, limits)
 
     output_data = read_stream(source, config, catalog, _A_PER_PARTITION_STATE, limits).record.data
     slices = output_data["slices"]
@@ -1143,7 +1140,7 @@ def test_read_source_single_page_single_slice(mock_http_stream):
 
     config = {"__injected_declarative_manifest": MANIFEST}
 
-    source = create_source(config, limits, post_resolve_manifest=True)
+    source = create_source(config, limits)
 
     output_data = read_stream(source, config, catalog, _A_PER_PARTITION_STATE, limits).record.data
     slices = output_data["slices"]
@@ -1229,7 +1226,7 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error
     test_manifest["streams"][0]["$parameters"]["url_base"] = url_base
     config = {"__injected_declarative_manifest": test_manifest}
 
-    source = create_source(config, limits, post_resolve_manifest=True)
+    source = create_source(config, limits)
 
     with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False):
         output_data = read_stream(
@@ -1325,7 +1322,7 @@ def test_handle_read_external_oauth_request(deployment_mode, token_url, expected
     )
     config = {"__injected_declarative_manifest": test_manifest}
 
-    source = create_source(config, limits, post_resolve_manifest=True)
+    source = create_source(config, limits)
 
     with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False):
         output_data = read_stream(