Feat: Promote get_source() args for docker_image and source_manifest out of experimental status (#294)

Author: aaronsteers

airbyte/experimental/__init__.py

@@ -1,6 +1,10 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 """Experimental features which may change.
 
+> **NOTE:**
+> The below "experimental" features are now "stable" and can be accessed directly from the
+`airbyte.get_source()` method.
+
 The experimental `get_source` implementation allows you to run sources
 using Docker containers. This feature is still in development and may
 change in the future.
@@ -34,7 +38,7 @@
 
 from __future__ import annotations
 
-from airbyte.sources.util import _get_source as get_source
+from airbyte.sources.util import get_source
 
 
 __all__ = [

airbyte/sources/util.py

@@ -48,9 +48,7 @@ def get_connector(
     )
 
 
-# This non-public function includes the `docker_image` parameter, which is not exposed in the
-# public API. See the `experimental` module for more info.
-def _get_source(  # noqa: PLR0912, PLR0913, PLR0915 # Too complex
+def get_source(  # noqa: PLR0912, PLR0913, PLR0915 # Too complex
     name: str,
     config: dict[str, Any] | None = None,
     *,
@@ -87,13 +85,14 @@ def _get_source(  # noqa: PLR0912, PLR0913, PLR0915 # Too complex
             (e.g. `my-image:latest`), the version will be appended as a tag (e.g. `my-image:0.1.0`).
         use_host_network: If set, along with docker_image, the connector will be executed using
             the host network. This is useful for connectors that need to access resources on
-            the host machine, such as a local database.
-        source_manifest: If set, the connector will be executed based on a declarative Yaml
-            source definition. This input can be `True` to auto-download the yaml spec, `dict`
-            to accept a Python dictionary as the manifest, `Path` to pull a manifest from
+            the host machine, such as a local database. This parameter is ignored when
+            `docker_image` is not set.
+        source_manifest: If set, the connector will be executed based on a declarative YAML
+            source definition. This input can be `True` to attempt to auto-download a YAML spec,
+            `dict` to accept a Python dictionary as the manifest, `Path` to pull a manifest from
             the local file system, or `str` to pull the definition from a web URL.
         install_if_missing: Whether to install the connector if it is not available locally. This
-            parameter is ignored when local_executable is set.
+            parameter is ignored when `local_executable` or `source_manifest` are set.
         install_root: (Optional.) The root directory where the virtual environment will be
             created. If not provided, the current working directory will be used.
     """
@@ -206,13 +205,15 @@ def _get_source(  # noqa: PLR0912, PLR0913, PLR0915 # Too complex
 
     if source_manifest:
         if source_manifest is True:
-            http_url = (
+            # Auto-set the manifest to a valid http address URL string
+            source_manifest = (
                 "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations"
                 f"/connectors/{name}/{name.replace('-', '_')}/manifest.yaml"
             )
-            print("Installing connector from YAML manifest:", http_url)
-            # Download the file
-            response = requests.get(http_url)
+        if isinstance(source_manifest, str):
+            print("Installing connector from YAML manifest:", source_manifest)
+            # Download the manifest file
+            response = requests.get(url=source_manifest)
             response.raise_for_status()  # Raise an exception if the download failed
 
             if "class_name:" in response.text:
@@ -225,7 +226,7 @@ def _get_source(  # noqa: PLR0912, PLR0913, PLR0915 # Too complex
                     ),
                     connector_name=name,
                     context={
-                        "manifest_url": http_url,
+                        "manifest_url": source_manifest,
                     },
                 )
 
@@ -235,10 +236,14 @@ def _get_source(  # noqa: PLR0912, PLR0913, PLR0915 # Too complex
                 raise exc.AirbyteConnectorInstallationError(
                     connector_name=name,
                     context={
-                        "manifest_url": http_url,
+                        "manifest_url": source_manifest,
                     },
                 ) from ex
 
+        if isinstance(source_manifest, Path):
+            source_manifest = cast(dict, yaml.safe_load(source_manifest.read_text()))
+
+        # Source manifest is a dict at this point
         return Source(
             name=name,
             config=config,
@@ -280,54 +285,6 @@ def _get_source(  # noqa: PLR0912, PLR0913, PLR0915 # Too complex
         raise
 
 
-# This thin wrapper exposes only non-experimental functions.
-# Aka, exclude the `docker_image` parameter for now.
-# See the `experimental` module for more info.
-def get_source(
-    name: str,
-    config: dict[str, Any] | None = None,
-    *,
-    streams: str | list[str] | None = None,
-    version: str | None = None,
-    pip_url: str | None = None,
-    local_executable: Path | str | None = None,
-    install_if_missing: bool = True,
-    install_root: Path | None = None,
-) -> Source:
-    """Get a connector by name and version.
-
-    Args:
-        name: connector name
-        config: connector config - if not provided, you need to set it later via the set_config
-            method.
-        streams: list of stream names to select for reading. If set to "*", all streams will be
-            selected. If not provided, you can set it later via the `select_streams()` or
-            `select_all_streams()` method.
-        version: connector version - if not provided, the currently installed version will be used.
-            If no version is installed, the latest available version will be used. The version can
-            also be set to "latest" to force the use of the latest available version.
-        pip_url: connector pip URL - if not provided, the pip url will be inferred from the
-            connector name.
-        local_executable: If set, the connector will be assumed to already be installed and will be
-            executed using this path or executable name. Otherwise, the connector will be installed
-            automatically in a virtual environment.
-        install_if_missing: Whether to install the connector if it is not available locally. This
-            parameter is ignored when local_executable is set.
-        install_root: (Optional.) The root directory where the virtual environment will be
-            created. If not provided, the current working directory will be used.
-    """
-    return _get_source(
-        name=name,
-        config=config,
-        streams=streams,
-        version=version,
-        pip_url=pip_url,
-        local_executable=local_executable,
-        install_if_missing=install_if_missing,
-        install_root=install_root,
-    )
-
-
 __all__ = [
     "get_source",
 ]

examples/run_declarative_manifest_source.py

@@ -12,7 +12,7 @@
 
 import yaml
 
-from airbyte.experimental import get_source
+from airbyte import get_source
 
 
 # Copy-pasted from the Builder "Yaml" view:

examples/run_downloadable_yaml_source.py

@@ -9,7 +9,7 @@
 from __future__ import annotations
 
 import airbyte as ab
-from airbyte.experimental import get_source
+from airbyte import get_source
 
 
 yaml_connectors: list[str] = ab.get_available_connectors(install_type="yaml")

examples/run_pokeapi.py

@@ -11,7 +11,7 @@
 from __future__ import annotations
 
 import airbyte as ab
-from airbyte.experimental import get_source
+from airbyte import get_source
 
 
 source = get_source(

tests/integration_tests/test_all_cache_types.py

@@ -14,8 +14,8 @@
 
 import airbyte as ab
 import pytest
+from airbyte import get_source
 from airbyte._executor import _get_bin_dir
-from airbyte.experimental import get_source
 from airbyte.progress import ReadProgress, progress
 
 # Product count is always the same, regardless of faker scale.

tests/integration_tests/test_lowcode_connectors.py

@@ -7,7 +7,7 @@
 import jsonschema
 import pytest
 from airbyte import exceptions as exc
-from airbyte.experimental import get_source
+from airbyte import get_source
 from airbyte.sources.registry import (
     _LOWCODE_CONNECTORS_404,
     _LOWCODE_CONNECTORS_FAILING_VALIDATION,

tests/unit_tests/test_lowcode_connectors.py

@@ -4,8 +4,8 @@
 from pathlib import Path
 
 import pytest
+from airbyte import get_source
 from airbyte._util.meta import is_windows
-from airbyte.experimental import get_source
 
 UNIT_TEST_DB_PATH: Path = Path(".cache") / "unit_tests" / "test_db.duckdb"