Merge branch 'main' into brian/remerge_concurrent_cdk_builder_changes

Author: brianjlai

.github/pr-welcome-internal.md

@@ -26,6 +26,7 @@ Airbyte Maintainers can execute the following slash commands on your PR:
 - `/autofix` - Fixes most formatting and linting issues
 - `/poetry-lock` - Updates poetry.lock file
 - `/test` - Runs connector tests with the updated CDK
+- `/poe build` - Regenerate git-committed build artifacts, such as the pydantic models which are generated from the manifest JSON schema in YAML.
 - `/poe <command>` - Runs any poe command in the CDK environment
 
 [📝 _Edit this welcome message._](https://github.com/airbytehq/airbyte-python-cdk/blob/main/.github/pr-welcome-internal.md)

airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -123,11 +123,18 @@ def connector_cli_group() -> None:
     multiple=True,
     help="Additional argument(s) to pass to pytest. Can be specified multiple times.",
 )
+@click.option(
+    "--no-creds",
+    is_flag=True,
+    default=False,
+    help="Skip tests that require credentials (marked with 'requires_creds').",
+)
 def connector_test(
     connector: str | Path | None = None,
     *,
     collect_only: bool = False,
     pytest_args: list[str] | None = None,
+    no_creds: bool = False,
 ) -> None:
     """Run connector tests.
 
@@ -147,6 +154,9 @@ def connector_test(
     if collect_only:
         pytest_args.append("--collect-only")
 
+    if no_creds:
+        pytest_args.extend(["-m", "not requires_creds"])
+
     run_connector_tests(
         connector_name=connector_name,
         connector_directory=connector_directory,

airbyte_cdk/cli/airbyte_cdk/_image.py

@@ -100,10 +100,17 @@ def build(
     "--image",
     help="Image to test, instead of building a new one.",
 )
+@click.option(
+    "--no-creds",
+    is_flag=True,
+    default=False,
+    help="Skip tests that require credentials (marked with 'requires_creds').",
+)
 def image_test(  # "image test" command
     connector: str | None = None,
     *,
     image: str | None = None,
+    no_creds: bool = False,
 ) -> None:
     """Test a connector Docker image.
 
@@ -124,7 +131,11 @@ def image_test(  # "image test" command
     connector_name, connector_directory = resolve_connector_name_and_directory(connector)
 
     # Select only tests with the 'image_tests' mark
-    pytest_args = ["-m", "image_tests"]
+    pytest_filter = "image_tests"
+    if no_creds:
+        pytest_filter += " and not requires_creds"
+
+    pytest_args = ["-m", pytest_filter]
     if not image:
         metadata_file_path: Path = connector_directory / "metadata.yaml"
         try:

airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -168,7 +168,13 @@ def _get_url(
             next_page_token=next_page_token,
         )
 
-        full_url = self._join_url(url_base, path) if url_base else url + path if path else url
+        full_url = (
+            self._join_url(url_base, path)
+            if url_base
+            else self._join_url(url, path)
+            if path
+            else url
+        )
 
         return full_url
 

airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -159,14 +159,7 @@ def path(
     ) -> Optional[str]:
         token = next_page_token.get("next_page_token") if next_page_token else None
         if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
-            # make additional interpolation context
-            interpolation_context = get_interpolation_context(
-                stream_state=stream_state,
-                stream_slice=stream_slice,
-                next_page_token=next_page_token,
-            )
-            # Replace url base to only return the path
-            return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+            return str(token)
         else:
             return None
 

airbyte_cdk/test/models/scenario.py

@@ -186,3 +186,8 @@ def with_expecting_success(self) -> ConnectorTestScenario:
             **self.model_dump(exclude={"status"}),
             status="succeed",
         )
+
+    @property
+    def requires_creds(self) -> bool:
+        """Return True if the scenario requires credentials to run."""
+        return bool(self.config_path and "secrets" in self.config_path.parts)

airbyte_cdk/test/standard_tests/pytest_hooks.py

@@ -161,14 +161,29 @@ class TestMyConnector(ConnectorTestSuiteBase):
         if test_class is None:
             return
 
-        # Get the 'scenarios' attribute from the class
+        # Check that the class is compatible with our test suite
         scenarios_attr = getattr(test_class, "get_scenarios", None)
         if scenarios_attr is None:
             raise ValueError(
                 f"Test class {test_class} does not have a 'scenarios' attribute. "
                 "Please define the 'scenarios' attribute in the test class."
             )
 
+        # Get the scenarios defined or discovered in the test class
         scenarios = test_class.get_scenarios()
-        ids = [str(scenario) for scenario in scenarios]
-        metafunc.parametrize("scenario", scenarios, ids=ids)
+
+        # Create pytest.param objects with special marks as needed
+        parametrized_scenarios = [
+            pytest.param(
+                scenario,
+                marks=[pytest.mark.requires_creds] if scenario.requires_creds else [],
+            )
+            for scenario in scenarios
+        ]
+
+        # Parametrize the 'scenario' argument with the scenarios
+        metafunc.parametrize(
+            "scenario",
+            parametrized_scenarios,
+            ids=[str(scenario) for scenario in scenarios],
+        )

docs/CONTRIBUTING.md

@@ -113,6 +113,7 @@ Only Airbyte CDK maintainers can run slash commands. The most common slash comma
 
 - `/autofix`- Corrects any linting or formatting issues and commits the change back to the repo.
 - `/poetry-lock` - Re-locks dependencies and updates the `poetry.lock` file, then commits the changes back to the repo. This is helpful after merging in updates from main, or when creating a PR in the browser - such as for version bumps or dependency updates directly in the PR.
+- `/poe ...` - Runs a Poe task in CI. If files are modified (e.g. as with `/poe build`), they will be committed back to your branch.
 
 The full list of available slash commands can be found in the [slash command dispatch file](https://github.com/airbytehq/airbyte-python-cdk/blob/main/.github/workflows/slash_command_dispatch.yml#L21-L25).
 

unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py

@@ -36,7 +36,7 @@
         (
             RequestPath(parameters={}),
             None,
-            "/next_url",
+            "https://airbyte.io/next_url",
             {"limit": 2},
             {},
             {},
@@ -128,7 +128,7 @@
         (
             RequestPath(parameters={}),
             None,
-            "/next_url",
+            "https://airbyte.io/next_url",
             {"limit": 2},
             {},
             {},
@@ -539,42 +539,3 @@ def test_path_returns_none_when_option_not_request_path() -> None:
     )
     result = paginator.path(next_page_token)
     assert result is None
-
-
-def test_path_with_additional_interpolation_context() -> None:
-    page_token_option = RequestPath(parameters={})
-    paginator = DefaultPaginator(
-        pagination_strategy=Mock(),
-        config={},
-        url_base="https://api.domain.com/{{ stream_slice['campaign_id'] }}",
-        parameters={},
-        page_token_option=page_token_option,
-    )
-    # define stream_state here
-    stream_state = {"state": "state_value"}
-    # define stream_slice here
-    stream_slice = StreamSlice(
-        partition={
-            "campaign_id": "123_abcd",
-        },
-        cursor_slice={
-            "start": "A",
-            "end": "B",
-        },
-        extra_fields={
-            "extra_field_A": "value_A",
-            "extra_field_B": "value_B",
-        },
-    )
-    # define next_page_token here
-    next_page_token = {
-        "next_page_token": "https://api.domain.com/123_abcd/some_next_page_token_here"
-    }
-
-    expected_after_interpolation = "/some_next_page_token_here"
-
-    assert expected_after_interpolation == paginator.path(
-        next_page_token=next_page_token,
-        stream_state=stream_state,
-        stream_slice=stream_slice,
-    )