add docker container ops

Author: aaronsteers

airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -90,7 +90,7 @@
 
 @click.group(
     name="connector",
-    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown)
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown) # type: ignore
 )
 def connector_cli_group() -> None:
     """Connector related commands."""
@@ -114,24 +114,43 @@ def connector_cli_group() -> None:
     default=False,
     help="Only collect tests, do not run them.",
 )
+@click.option(
+    "--use-docker-image",
+    # is_flag=True,
+    default=False,
+    type=str,
+    help="Run tests via docker.",
+    callback=lambda ctx, param, value: (
+        "true" if value is True else (value if isinstance(value, str) else False)
+    ),
+)
 def test(
     connector_name: str | None = None,
     connector_directory: Path | None = None,
     *,
     collect_only: bool = False,
+    use_docker_image: str | bool = False,
 ) -> None:
     """Run connector tests.
 
     This command runs the standard connector tests for a specific connector.
 
     If no connector name or directory is provided, we will look within the current working
     directory. If the current working directory is not a connector directory (e.g. starting
-    with 'source-') and no connector name or path is provided, the process will fail.
+    with 'source-' or 'destination-') and no connector name or path is provided, the process
+    will fail.
     """
+    if isinstance(
+        use_docker_image,
+        str,
+    ) and use_docker_image.lower() in {"true", "false"}:
+        use_docker_image = bool(use_docker_image)
+
     if pytest is None:
         raise ImportError(
             "pytest is not installed. Please install pytest to run the connector tests."
         )
+
     click.echo("Connector test command executed.")
     connector_name, connector_directory = resolve_connector_name_and_directory(
         connector_name=connector_name,
@@ -159,9 +178,18 @@ def test(
     test_file_path.parent.mkdir(parents=True, exist_ok=True)
     test_file_path.write_text(file_text)
 
+    pytest_args.append("-p airbyte_cdk.test.standard_tests.pytest_hooks")
+
     if collect_only:
         pytest_args.append("--collect-only")
 
+    if use_docker_image:
+        pytest_args.append("--use-docker-image")
+        if isinstance(use_docker_image, str):
+            pytest_args.append(
+                use_docker_image,
+            )
+
     pytest_args.append(str(test_file_path))
     click.echo(f"Running tests from connector directory: {connector_directory}...")
     click.echo(f"Test file: {test_file_path}")

airbyte_cdk/cli/airbyte_cdk/_image.py

@@ -64,21 +64,11 @@ def build(
         connector_directory=connector_directory,
     )
 
-    metadata_file_path: Path = connector_directory / "metadata.yaml"
-    try:
-        metadata = MetadataFile.from_file(metadata_file_path)
-    except (FileNotFoundError, ValueError) as e:
-        click.echo(
-            f"Error loading metadata file '{metadata_file_path}': {e!s}",
-            err=True,
-        )
-        sys.exit(1)
-    click.echo(f"Building Image for Connector: {metadata.data.dockerRepository}:{tag}")
+    click.echo(f"Building '{tag}' Image for Connector '{connector_name}':")
     try:
         build_connector_image(
             connector_directory=connector_directory,
             connector_name=connector_name,
-            metadata=metadata,
             tag=tag,
             no_verify=no_verify,
         )

airbyte_cdk/test/standard_tests/_job_runner.py

@@ -4,15 +4,18 @@
 import logging
 import tempfile
 import uuid
+from abc import abstractmethod
+from collections.abc import Callable
 from dataclasses import asdict
 from pathlib import Path
-from typing import Any, Callable, Literal
+from typing import Any, Literal
 
 import orjson
 from typing_extensions import Protocol, runtime_checkable
 
 from airbyte_cdk.models import (
     ConfiguredAirbyteCatalog,
+    ConnectorSpecification,
     Status,
 )
 from airbyte_cdk.test import entrypoint_wrapper
@@ -50,8 +53,10 @@ class IConnector(Protocol):
     directly on the connector (which doesn't yet exist).
     """
 
-    def spec(self, logger: logging.Logger) -> Any:
+    @abstractmethod
+    def spec(self, logger: logging.Logger) -> ConnectorSpecification:
         """Connectors should have a `spec` method."""
+        ...
 
 
 def run_test_job(

airbyte_cdk/test/standard_tests/connector_base.py

@@ -8,10 +8,10 @@
 import inspect
 import os
 import sys
-from collections.abc import Callable
 from pathlib import Path
-from typing import cast
+from typing import Literal, cast
 
+import pytest
 import yaml
 from boltons.typeutils import classproperty
 
@@ -21,6 +21,7 @@
 )
 from airbyte_cdk.test import entrypoint_wrapper
 from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
+from airbyte_cdk.test.standard_tests.docker_connectors import DockerConnector
 from airbyte_cdk.test.standard_tests.models import (
     ConnectorTestScenario,
 )
@@ -30,19 +31,42 @@
 )
 
 
+@pytest.fixture
+def use_docker_image(request: pytest.FixtureRequest) -> str | bool:
+    """Fixture to determine if a Docker image should be used for the test."""
+    return request.config.getoption("use_docker_image")
+
+
 class ConnectorTestSuiteBase(abc.ABC):
     """Base class for connector test suites."""
 
-    connector: type[IConnector] | Callable[[], IConnector] | None  # type: ignore [reportRedeclaration]
+    connector_class: type[IConnector] | None = None
     """The connector class or a factory function that returns an scenario of IConnector."""
 
-    @classproperty  # type: ignore [no-redef]
-    def connector(cls) -> type[IConnector] | Callable[[], IConnector] | None:
+    @classmethod
+    def get_test_class_dir(cls) -> Path:
+        """Get the file path that contains the class."""
+        module = sys.modules[cls.__module__]
+        # Get the directory containing the test file
+        return Path(inspect.getfile(module)).parent
+
+    @classmethod
+    def create_connector(
+        cls,
+        scenario: ConnectorTestScenario,
+        use_docker_image: str | bool,
+    ) -> IConnector:
+        """Instantiate the connector class."""
         """Get the connector class for the test suite.
 
         This assumes a python connector and should be overridden by subclasses to provide the
         specific connector class to be tested.
         """
+        if use_docker_image:
+            return cls.create_docker_connector(
+                docker_image=use_docker_image,
+            )
+
         connector_root = cls.get_connector_root_dir()
         connector_name = connector_root.absolute().name
 
@@ -65,7 +89,10 @@ def connector(cls) -> type[IConnector] | Callable[[], IConnector] | None:
 
         # Dynamically get the class from the module
         try:
-            return cast(type[IConnector], getattr(module, expected_class_name))
+            return cast(
+                type[IConnector],
+                getattr(module, expected_class_name),
+            )()
         except AttributeError as e:
             # We did not find it based on our expectations, so let's check if we can find it
             # with a case-insensitive match.
@@ -77,21 +104,13 @@ def connector(cls) -> type[IConnector] | Callable[[], IConnector] | None:
                 raise ImportError(
                     f"Module '{expected_module_name}' does not have a class named '{expected_class_name}'."
                 ) from e
-            return cast(type[IConnector], getattr(module, matching_class_name))
+            return cast(
+                type[IConnector],
+                getattr(module, matching_class_name),
+            )()
 
-    @classmethod
-    def get_test_class_dir(cls) -> Path:
-        """Get the file path that contains the class."""
-        module = sys.modules[cls.__module__]
-        # Get the directory containing the test file
-        return Path(inspect.getfile(module)).parent
+        _ = scenario
 
-    @classmethod
-    def create_connector(
-        cls,
-        scenario: ConnectorTestScenario,
-    ) -> IConnector:
-        """Instantiate the connector class."""
         connector = cls.connector  # type: ignore
         if connector:
             if callable(connector) or isinstance(connector, type):
@@ -105,15 +124,46 @@ def create_connector(
             "override `cls.create_connector()` to define a custom initialization process."
         )
 
+    @classmethod
+    def create_docker_connector(
+        cls,
+        docker_image: str | Literal[True],
+    ) -> IConnector:
+        """Create a connector instance using Docker."""
+        if not docker_image:
+            raise ValueError("Docker image is required to create a Docker connector.")
+
+        # Create the connector object by building the connector
+        if docker_image is True:
+            return DockerConnector.from_connector_directory(
+                connector_directory=cls.get_connector_root_dir(),
+            )
+
+        if not isinstance(docker_image, str):
+            raise ValueError(
+                "Expected `docker_image` to be 'True' or of type `str`. "
+                f"Type found: {type(docker_image).__name__}"
+            )
+
+        # Create the connector object using the provided Docker image
+        return DockerConnector(
+            connector_name=cls.get_connector_root_dir().name,
+            docker_image=docker_image,
+        )
+
     # Test Definitions
 
     def test_check(
         self,
         scenario: ConnectorTestScenario,
+        use_docker_image: str | bool,
     ) -> None:
         """Run `connection` acceptance tests."""
         result: entrypoint_wrapper.EntrypointOutput = run_test_job(
-            self.create_connector(scenario),
+            self.create_connector(
+                scenario,
+                use_docker_image=use_docker_image,
+            ),
             "check",
             test_scenario=scenario,
         )

airbyte_cdk/test/standard_tests/declarative_sources.py

@@ -65,6 +65,7 @@ def components_py_path(cls) -> Path | None:
     def create_connector(
         cls,
         scenario: ConnectorTestScenario,
+        use_docker_image: str | bool,
     ) -> IConnector:
         """Create a connector scenario for the test suite.
 
@@ -73,6 +74,11 @@ def create_connector(
 
         Subclasses should not need to override this method.
         """
+        if use_docker_image:
+            return cls.create_docker_connector(
+                docker_image=use_docker_image,
+            )
+
         config: dict[str, Any] = scenario.get_config_dict()
 
         manifest_dict = yaml.safe_load(cls.manifest_yaml_path.read_text())