Merge branch 'main' into pnilan/feat/extend-spec-class-for-config-migrations

Author: pnilan

.coveragerc

@@ -9,3 +9,13 @@ omit =
 
     # omit as unimplemented
     airbyte_cdk/base_python/cdk/streams/auth/jwt.py
+
+    # omit temporary files and test files
+    /tmp/tmp*.py
+    unit_tests/*
+
+[paths]
+# Reconcile file paths
+source =
+    ./
+    /tmp/

.github/workflows/auto_merge_notification.yml

@@ -0,0 +1,31 @@
+# When a PR is has the auto-merge feature enabled or disabled, this workflow adds or updates
+# warning text at the bottom of the PR description.
+
+name: "Add Auto-Merge Notification Text"
+on:
+  pull_request:
+    types: [auto_merge_enabled, auto_merge_disabled]
+
+jobs:
+  update-description:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Add Auto-Merge Notice
+        if: github.event.action == 'auto_merge_enabled'
+        uses: bcgov/[email protected]
+        with:
+          add_markdown: |
+            > [!IMPORTANT]
+            > **Auto-merge enabled.**
+            > 
+            > _This PR is set to merge automatically when all requirements are met._
+
+      - name: Remove Auto-Merge Notice
+        if: github.event.action == 'auto_merge_disabled'
+        uses: bcgov/[email protected]
+        with:
+          add_markdown: |
+            > [!NOTE]
+            > **Auto-merge may have been disabled. Please check the PR status to confirm.**

.pre-commit-config.yaml

@@ -3,15 +3,14 @@
 exclude: |
   (?x)(
     # Python/system files
-    ^.*/__init__\.py$|
     ^.*?/\.venv/.*$|
     ^.*?/node_modules/.*$|
-    
+
     # Generated/test files
     ^.*?/\.pytest_cache/.*$|
     ^.*?/__pycache__/.*$|
     ^.*?/\.mypy_cache/.*$|
-    ^.*?/\.ruff_cache/.*$
+    ^.*?/\.ruff_cache/.*$|
 
     # Package management
     ^.*?/poetry\.lock$|
@@ -22,7 +21,7 @@ exclude: |
     ^.*?/build/.*$|
     ^.*?/dist/.*$|
     ^.*?/\.coverage$|
-    ^.*?/coverage\.xml$|
+    ^.*?/coverage\.xml$
   )
 
 repos:
@@ -38,18 +37,15 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.11.5
     hooks:
-      # Run the linter with repo-defined settings
       - id: ruff
         args: [--fix]
-
-      # Run the formatter with repo-defined settings
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.0.3
     hooks:
       - id: prettier
-        types_or: [json, yaml]
+        args: [--write]
         additional_dependencies:
           - [email protected]
 
@@ -61,3 +57,20 @@ repos:
         language: golang
         additional_dependencies: [github.com/google/[email protected]]
         files: \.py$
+
+      - id: poetry-check
+        name: Check Poetry lockfile
+        entry: poetry check
+        language: system
+        pass_filenames: false
+        always_run: true
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        args: [--config-file=mypy.ini, --show-column-numbers]
+        files: ^airbyte_cdk/
+        pass_filenames: true

airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -39,6 +39,7 @@
 """
 
 import os
+import sys
 from pathlib import Path
 from types import ModuleType
 
@@ -166,10 +167,11 @@ def test(
     click.echo(f"Collect only: {collect_only}")
     click.echo(f"Pytest args: {pytest_args}")
     click.echo("Invoking Pytest...")
-    pytest.main(
+    exit_code = pytest.main(
         pytest_args,
         plugins=[],
     )
+    sys.exit(exit_code)
 
 
 __all__ = [

airbyte_cdk/cli/airbyte_cdk/_secrets.py

@@ -49,7 +49,12 @@
     resolve_connector_name_and_directory,
 )
 
-AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing"
+GCP_PROJECT_ID: str = os.environ.get("GCP_PROJECT_ID", "") or "dataline-integration-testing"
+# We put the `or` outside the `get()` because we want the `GCP_PROJECT_ID`
+# env var to be ignored if it contains an empty string, such as in CI where the
+# workflow might set it to a value that is itself actually missing or unset.
+"""The GCP project ID to use for fetching integration test secrets."""
+
 CONNECTOR_LABEL = "connector"
 GLOBAL_MASK_KEYS_URL = "https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml"
 
@@ -83,8 +88,11 @@ def secrets_cli_group() -> None:
 @click.option(
     "--gcp-project-id",
     type=str,
-    default=AIRBYTE_INTERNAL_GCP_PROJECT,
-    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+    default=GCP_PROJECT_ID,
+    help=(
+        "GCP project ID for retrieving integration tests credentials. "
+        "Defaults to the value of the `GCP_PROJECT_ID` environment variable, if set."
+    ),
 )
 @click.option(
     "--print-ci-secrets-masks",
@@ -95,7 +103,7 @@ def secrets_cli_group() -> None:
 )
 def fetch(
     connector: str | Path | None = None,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    gcp_project_id: str = GCP_PROJECT_ID,
     print_ci_secrets_masks: bool = False,
 ) -> None:
     """Fetch secrets for a connector from Google Secret Manager.
@@ -192,41 +200,41 @@ def fetch(
 
 
 @secrets_cli_group.command("list")
-@click.option(
-    "--connector-name",
+@click.argument(
+    "connector",
+    required=False,
     type=str,
-    help="Name of the connector to fetch secrets for. Ignored if --connector-directory is provided.",
-)
-@click.option(
-    "--connector-directory",
-    type=click.Path(exists=True, file_okay=False, path_type=Path),
-    help="Path to the connector directory.",
+    metavar="[CONNECTOR]",
 )
 @click.option(
     "--gcp-project-id",
     type=str,
-    default=AIRBYTE_INTERNAL_GCP_PROJECT,
-    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+    default=GCP_PROJECT_ID,
+    help=(
+        "GCP project ID for retrieving integration tests credentials. "
+        "Defaults to the value of the `GCP_PROJECT_ID` environment variable, if set."
+    ),
 )
 def list_(
-    connector_name: str | None = None,
-    connector_directory: Path | None = None,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    connector: str | Path | None = None,
+    *,
+    gcp_project_id: str = GCP_PROJECT_ID,
 ) -> None:
     """List secrets for a connector from Google Secret Manager.
 
     This command fetches secrets for a connector from Google Secret Manager and prints
     them as a table.
 
+    [CONNECTOR] can be a connector name (e.g. 'source-pokeapi'), a path to a connector directory, or omitted to use the current working directory.
+    If a string containing '/' is provided, it is treated as a path. Otherwise, it is treated as a connector name.
+
     If no connector name or directory is provided, we will look within the current working
     directory. If the current working directory is not a connector directory (e.g. starting
     with 'source-') and no connector name or path is provided, the process will fail.
     """
     click.echo("Scanning secrets...", err=True)
 
-    connector_name = connector_name or resolve_connector_name(
-        connector_directory=connector_directory or Path().resolve().absolute(),
-    )
+    connector_name, _ = resolve_connector_name_and_directory(connector)
     secrets: list[Secret] = _fetch_secret_handles(  # type: ignore
         connector_name=connector_name,
         gcp_project_id=gcp_project_id,
@@ -303,7 +311,7 @@ def _get_secret_url(secret_name: str, gcp_project_id: str) -> str:
 
 def _fetch_secret_handles(
     connector_name: str,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    gcp_project_id: str = GCP_PROJECT_ID,
 ) -> list["Secret"]:  # type: ignore
     """Fetch secrets from Google Secret Manager."""
     if not secretmanager:

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -16,15 +16,17 @@
 
 from __future__ import annotations
 
+import argparse
 import json
 import pkgutil
 import sys
 import traceback
-from collections.abc import Mapping
+from collections.abc import MutableMapping
 from pathlib import Path
 from typing import Any, cast
 
 import orjson
+import yaml
 
 from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.models import (
@@ -54,7 +56,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
     def __init__(
         self,
         catalog: ConfiguredAirbyteCatalog | None,
-        config: Mapping[str, Any] | None,
+        config: MutableMapping[str, Any] | None,
         state: TState,
         **kwargs: Any,
     ) -> None:
@@ -91,7 +93,8 @@ def handle_command(args: list[str]) -> None:
 
 def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
     try:
-        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        parsed_args = AirbyteEntrypoint.parse_args(args)
+        config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
         return SourceLocalYaml(config=config, catalog=catalog, state=state)
     except Exception as error:
         print(
@@ -162,21 +165,40 @@ def create_declarative_source(
     connector builder.
     """
     try:
-        config: Mapping[str, Any] | None
+        config: MutableMapping[str, Any] | None
         catalog: ConfiguredAirbyteCatalog | None
         state: list[AirbyteStateMessage]
-        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
-        if config is None or "__injected_declarative_manifest" not in config:
+
+        parsed_args = AirbyteEntrypoint.parse_args(args)
+        config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
+
+        if config is None:
+            raise ValueError(
+                "Invalid config: `__injected_declarative_manifest` should be provided at the root "
+                "of the config or using the --manifest-path argument."
+            )
+
+        # If a manifest_path is provided in the args, inject it into the config
+        if hasattr(parsed_args, "manifest_path") and parsed_args.manifest_path:
+            injected_manifest = _parse_manifest_from_file(parsed_args.manifest_path)
+            if injected_manifest:
+                config["__injected_declarative_manifest"] = injected_manifest
+
+        if "__injected_declarative_manifest" not in config:
             raise ValueError(
                 "Invalid config: `__injected_declarative_manifest` should be provided at the root "
-                f"of the config but config only has keys: {list(config.keys() if config else [])}"
+                "of the config or using the --manifest-path argument. "
+                f"Config only has keys: {list(config.keys() if config else [])}"
             )
         if not isinstance(config["__injected_declarative_manifest"], dict):
             raise ValueError(
                 "Invalid config: `__injected_declarative_manifest` should be a dictionary, "
                 f"but got type: {type(config['__injected_declarative_manifest'])}"
             )
 
+        if hasattr(parsed_args, "components_path") and parsed_args.components_path:
+            _register_components_from_file(parsed_args.components_path)
+
         return ConcurrentDeclarativeSource(
             config=config,
             catalog=catalog,
@@ -205,13 +227,12 @@ def create_declarative_source(
 
 
 def _parse_inputs_into_config_catalog_state(
-    args: list[str],
+    parsed_args: argparse.Namespace,
 ) -> tuple[
-    Mapping[str, Any] | None,
+    MutableMapping[str, Any] | None,
     ConfiguredAirbyteCatalog | None,
     list[AirbyteStateMessage],
 ]:
-    parsed_args = AirbyteEntrypoint.parse_args(args)
     config = (
         ConcurrentDeclarativeSource.read_config(parsed_args.config)
         if hasattr(parsed_args, "config")
@@ -231,6 +252,44 @@ def _parse_inputs_into_config_catalog_state(
     return config, catalog, state
 
 
+def _parse_manifest_from_file(filepath: str) -> dict[str, Any] | None:
+    """Extract and parse a manifest file specified in the args."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as manifest_file:
+            manifest_content = yaml.safe_load(manifest_file)
+            if manifest_content is None:
+                raise ValueError(f"Manifest file at {filepath} is empty")
+            if not isinstance(manifest_content, dict):
+                raise ValueError(f"Manifest must be a dictionary, got {type(manifest_content)}")
+            return manifest_content
+    except Exception as error:
+        raise ValueError(f"Failed to load manifest file from {filepath}: {error}")
+
+
+def _register_components_from_file(filepath: str) -> None:
+    """Load and register components from a Python file specified in the args."""
+    import importlib.util
+    import sys
+
+    components_path = Path(filepath)
+
+    module_name = "components"
+    sdm_module_name = "source_declarative_manifest.components"
+
+    # Create module spec
+    spec = importlib.util.spec_from_file_location(module_name, components_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Could not load module from {components_path}")
+
+    # Create module and execute code, registering the module before executing its code
+    # To avoid issues with dataclasses that look up the module
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    sys.modules[sdm_module_name] = module
+
+    spec.loader.exec_module(module)
+
+
 def run() -> None:
     args: list[str] = sys.argv[1:]
     handle_command(args)