Merge branch 'aj/feat/add-standard-tests-cli' into devin/1744841809-add-build-command

Author: aaronsteers

.gitignore

@@ -15,3 +15,4 @@ dist
 .idea
 .vscode
 **/__pycache__
+.tmp

airbyte_cdk/cli/airbyte_cdk/__init__.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""CLI commands for `airbyte-cdk`.
+
+This CLI interface allows you to interact with your connector, including
+testing and running commands.
+
+**Basic Usage:**
+
+```bash
+airbyte-cdk --help
+airbyte-cdk --version
+airbyte-cdk connector --help
+airbyte-cdk manifest --help
+```
+
+**Running Statelessly:**
+
+You can run the latest version of this CLI, from any machine, using `pipx` or `uvx`:
+
+```bash
+# Run the latest version of the CLI:
+pipx run airbyte-cdk connector --help
+uvx airbyte-cdk connector --help
+
+# Run from a specific CDK version:
+pipx run airbyte-cdk==6.5.1 connector --help
+uvx airbyte-cdk==6.5.1 connector --help
+```
+
+**Running within your virtualenv:**
+
+You can also run from your connector's virtualenv:
+
+```bash
+poetry run airbyte-cdk connector --help
+```
+
+"""
+
+from typing import cast
+
+import rich_click as click
+
+from airbyte_cdk.cli.airbyte_cdk._connector import connector_cli_group
+from airbyte_cdk.cli.airbyte_cdk._image import image_cli_group
+from airbyte_cdk.cli.airbyte_cdk._manifest import manifest_cli_group
+from airbyte_cdk.cli.airbyte_cdk._version import print_version
+
+
+@click.group(
+    help=__doc__.replace("\n", "\n\n"),  # Workaround to format help text correctly
+    invoke_without_command=True,
+)
+@click.option(
+    "--version",
+    is_flag=True,
+    help="Show the version of the Airbyte CDK.",
+)
+@click.pass_context
+def cli(
+    ctx: click.Context,
+    version: bool,
+) -> None:
+    """Airbyte CDK CLI.
+
+    Help text is provided from the file-level docstring.
+    """
+    if version:
+        print_version(short=False)
+        ctx.exit()
+
+    if ctx.invoked_subcommand is None:
+        # If no subcommand is provided, show the help message.
+        click.echo(ctx.get_help())
+        ctx.exit()
+
+
+cli.add_command(connector_cli_group)
+cli.add_command(manifest_cli_group)
+cli.add_command(image_cli_group)
+
+
+if __name__ == "__main__":
+    cli()

airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -0,0 +1,188 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""CLI command for `airbyte-cdk`."""
+
+USAGE = """CLI command for `airbyte-cdk`.
+
+This CLI interface allows you to interact with your connector, including
+testing and running commands.
+
+**Basic Usage:**
+
+```bash
+airbyte-cdk --help
+airbyte-cdk connector --help
+airbyte-cdk manifest --help
+```
+
+**Running Statelessly:**
+
+You can run the latest version of this CLI, from any machine, using `pipx` or `uvx`:
+
+```bash
+# Run the latest version of the CLI:
+pipx run airbyte-cdk connector --help
+uvx airbyte-cdk connector --help
+
+# Run from a specific CDK version:
+pipx run airbyte-cdk==6.5.1 connector --help
+uvx airbyte-cdk==6.5.1 connector --help
+```
+
+**Running within your virtualenv:**
+
+You can also run from your connector's virtualenv:
+
+```bash
+poetry run airbyte-cdk connector --help
+```
+
+"""
+
+import os
+from pathlib import Path
+from types import ModuleType
+
+import rich_click as click
+
+# from airbyte_cdk.test.standard_tests import pytest_hooks
+from airbyte_cdk.test.standard_tests.test_resources import find_connector_root_from_name
+from airbyte_cdk.test.standard_tests.util import create_connector_test_suite
+
+click.rich_click.TEXT_MARKUP = "markdown"
+
+pytest: ModuleType | None
+try:
+    import pytest
+except ImportError:
+    pytest = None
+    # Handle the case where pytest is not installed.
+    # This prevents import errors when running the script without pytest installed.
+    # We will raise an error later if pytest is required for a given command.
+
+
+TEST_FILE_TEMPLATE = '''
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""FAST Airbyte Standard Tests for the source_pokeapi_w_components source."""
+
+#from airbyte_cdk.test.standard_tests import {base_class_name}
+from airbyte_cdk.test.standard_tests.util import create_connector_test_suite
+from pathlib import Path
+
+pytest_plugins = [
+    "airbyte_cdk.test.standard_tests.pytest_hooks",
+]
+
+TestSuite = create_connector_test_suite(
+    connector_directory=Path(),
+)
+
+# class TestSuite({base_class_name}):
+#     """Test suite for the source_pokeapi_w_components source.
+
+#     This class inherits from SourceTestSuiteBase and implements all of the tests in the suite.
+
+#     As long as the class name starts with "Test", pytest will automatically discover and run the
+#     tests in this class.
+#     """
+'''
+
+
+@click.group(name="connector")
+def connector_cli_group() -> None:
+    """Connector related commands."""
+    pass
+
+
+@connector_cli_group.command()
+@click.option(
+    "--connector-name",
+    type=str,
+    # help="Name of the connector to test. Ignored if --connector-directory is provided.",
+)
+@click.option(
+    "--connector-directory",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    # help="Path to the connector directory.",
+)
+@click.option(
+    "--collect-only",
+    is_flag=True,
+    default=False,
+    help="Only collect tests, do not run them.",
+)
+def test(
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+    *,
+    collect_only: bool = False,
+) -> None:
+    """Run connector tests.
+
+    This command runs the standard connector tests for a specific connector.
+
+    If no connector name or directory is provided, we will look within the current working
+    directory. If the current working directory is not a connector directory (e.g. starting
+    with 'source-') and no connector name or path is provided, the process will fail.
+    """
+    if pytest is None:
+        raise ImportError(
+            "pytest is not installed. Please install pytest to run the connector tests."
+        )
+    click.echo("Connector test command executed.")
+    if not connector_name and not connector_directory:
+        cwd = Path().resolve().absolute()
+        if cwd.name.startswith("source-") or cwd.name.startswith("destination-"):
+            connector_name = cwd.name
+            connector_directory = cwd
+        else:
+            raise ValueError(
+                "Either connector_name or connector_directory must be provided if not "
+                "running from a connector directory."
+            )
+
+    if connector_directory:
+        connector_directory = connector_directory.resolve().absolute()
+    elif connector_name:
+        connector_directory = find_connector_root_from_name(connector_name)
+    else:
+        raise ValueError("Either connector_name or connector_directory must be provided.")
+
+    connector_test_suite = create_connector_test_suite(
+        connector_name=connector_name if not connector_directory else None,
+        connector_directory=connector_directory,
+    )
+
+    pytest_args: list[str] = []
+    if connector_directory:
+        pytest_args.append(f"--rootdir={connector_directory}")
+        os.chdir(str(connector_directory))
+    else:
+        print("No connector directory provided. Running tests in the current directory.")
+
+    file_text = TEST_FILE_TEMPLATE.format(
+        base_class_name=connector_test_suite.__bases__[0].__name__,
+        connector_directory=str(connector_directory),
+    )
+    test_file_path = Path() / ".tmp" / "integration_tests/test_airbyte_standards.py"
+    test_file_path = test_file_path.resolve().absolute()
+    test_file_path.parent.mkdir(parents=True, exist_ok=True)
+    test_file_path.write_text(file_text)
+
+    if collect_only:
+        pytest_args.append("--collect-only")
+
+    pytest_args.append(str(test_file_path))
+    click.echo(f"Running tests from connector directory: {connector_directory}...")
+    click.echo(f"Test file: {test_file_path}")
+    click.echo(f"Collect only: {collect_only}")
+    click.echo(f"Pytest args: {pytest_args}")
+    click.echo("Invoking Pytest...")
+    pytest.main(
+        pytest_args,
+        plugins=[],
+    )
+
+
+__all__ = [
+    "connector_cli_group",
+]

airbyte_cdk/cli/airbyte_cdk/_image.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Docker image commands."""
+
+import click
+
+
+@click.group(name="image")
+def image_cli_group() -> None:
+    """Docker image commands."""
+    pass
+
+
+__all__ = [
+    "image_cli_group",
+]

airbyte_cdk/cli/airbyte_cdk/_manifest.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Manifest related commands."""
+
+import click
+
+
+@click.group(name="manifest")
+def manifest_cli_group() -> None:
+    """Manifest related commands."""
+    pass
+
+
+__all__ = [
+    "manifest_cli_group",
+]

airbyte_cdk/cli/airbyte_cdk/_version.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Version information for the Airbyte CDK CLI."""
+
+from airbyte_cdk import __version__
+
+
+def print_version(short: bool = False) -> None:
+    """Print the version of the Airbyte CDK CLI."""
+
+    if short:
+        print(__version__)
+    else:
+        print(f"Airbyte CDK version: {__version__}")

airbyte_cdk/sources/declarative/auth/oauth.py

@@ -239,8 +239,8 @@ def get_token_expiry_date(self) -> AirbyteDateTime:
     def _has_access_token_been_initialized(self) -> bool:
         return self._access_token is not None
 
-    def set_token_expiry_date(self, value: Union[str, int]) -> None:
-        self._token_expiry_date = self._parse_token_expiration_date(value)
+    def set_token_expiry_date(self, value: AirbyteDateTime) -> None:
+        self._token_expiry_date = value
 
     def get_assertion_name(self) -> str:
         return self.assertion_name

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -19,7 +19,10 @@
 from airbyte_cdk.sources.declarative.extractors.record_filter import (
     ClientSideIncrementalRecordFilterDecorator,
 )
-from airbyte_cdk.sources.declarative.incremental import ConcurrentPerPartitionCursor
+from airbyte_cdk.sources.declarative.incremental import (
+    ConcurrentPerPartitionCursor,
+    GlobalSubstreamCursor,
+)
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
 from airbyte_cdk.sources.declarative.incremental.per_partition_with_global import (
     PerPartitionWithGlobalCursor,
@@ -361,7 +364,8 @@ def _group_streams(
                     == DatetimeBasedCursorModel.__name__
                     and hasattr(declarative_stream.retriever, "stream_slicer")
                     and isinstance(
-                        declarative_stream.retriever.stream_slicer, PerPartitionWithGlobalCursor
+                        declarative_stream.retriever.stream_slicer,
+                        (GlobalSubstreamCursor, PerPartitionWithGlobalCursor),
                     )
                 ):
                     stream_state = self._connector_state_manager.get_stream_state(

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -1439,7 +1439,9 @@ def create_concurrent_cursor_from_perpartition_cursor(
         stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
 
         # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state
-        use_global_cursor = isinstance(partition_router, GroupingPartitionRouter)
+        use_global_cursor = isinstance(
+            partition_router, GroupingPartitionRouter
+        ) or component_definition.get("global_substream_cursor", False)
 
         # Return the concurrent cursor and state converter
         return ConcurrentPerPartitionCursor(

airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py

@@ -52,8 +52,10 @@ def get_request_property_chunks(
         chunk_size = 0
         for property_field in property_fields:
             # If property_limit_type is not defined, we default to property_count which is just an incrementing count
+            # todo: Add ability to specify parameter delimiter representation and take into account in property_field_size
             property_field_size = (
                 len(property_field)
+                + 1  # The +1 represents the extra character for the delimiter in between properties
                 if self.property_limit_type == PropertyLimitType.characters
                 else 1
             )