merge with main

Author: lmossman

.coveragerc

@@ -9,3 +9,13 @@ omit =
 
     # omit as unimplemented
     airbyte_cdk/base_python/cdk/streams/auth/jwt.py
+
+    # omit temporary files and test files
+    /tmp/tmp*.py
+    unit_tests/*
+
+[paths]
+# Reconcile file paths
+source =
+    ./
+    /tmp/

.github/workflows/auto_merge_notification.yml

@@ -0,0 +1,31 @@
+# When a PR is has the auto-merge feature enabled or disabled, this workflow adds or updates
+# warning text at the bottom of the PR description.
+
+name: "Add Auto-Merge Notification Text"
+on:
+  pull_request:
+    types: [auto_merge_enabled, auto_merge_disabled]
+
+jobs:
+  update-description:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Add Auto-Merge Notice
+        if: github.event.action == 'auto_merge_enabled'
+        uses: bcgov/[email protected]
+        with:
+          add_markdown: |
+            > [!IMPORTANT]
+            > **Auto-merge enabled.**
+            > 
+            > _This PR is set to merge automatically when all requirements are met._
+
+      - name: Remove Auto-Merge Notice
+        if: github.event.action == 'auto_merge_disabled'
+        uses: bcgov/[email protected]
+        with:
+          add_markdown: |
+            > [!NOTE]
+            > **Auto-merge may have been disabled. Please check the PR status to confirm.**

.github/workflows/pytest_fast.yml

@@ -70,6 +70,15 @@ jobs:
           poetry run coverage html -d htmlcov
           poetry run coverage xml -o htmlcov/coverage.xml
 
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always() && !cancelled()
+        continue-on-error: true
+        with:
+          check_name: "PyTest Results (Fast)"
+          large_files: true
+          files: |
+            build/test-results/**/*.xml
       - name: Upload coverage to GitHub Artifacts
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/pytest_matrix.yml

@@ -93,6 +93,15 @@ jobs:
           poetry run coverage html -d htmlcov
           poetry run coverage xml -o htmlcov/coverage.xml
 
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always() && !cancelled() && steps.changes.outputs.src == 'true' && matrix.python-version == '3.11'
+        continue-on-error: true
+        with:
+          check_name: "PyTest Results (Full)"
+          large_files: true
+          files: |
+            build/test-results/**/*.xml
       - name: Upload coverage to GitHub Artifacts
         if: always() && steps.changes.outputs.src == 'true'
         uses: actions/upload-artifact@v4

.github/workflows/test-command.yml

@@ -113,6 +113,15 @@ jobs:
           --verbose
           -m "flaky and not super_slow"
 
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always() && !cancelled() && matrix.python-version == '3.11'
+        continue-on-error: true
+        with:
+          check_name: "PyTest Results (On-Demand)"
+          large_files: true
+          files: |
+            build/test-results/**/*.xml
       - name: Post CI Success to GitHub
         run: |
           curl --request POST \

.gitignore

@@ -1,3 +1,7 @@
+# Build and test results
+build
+test-results
+
 # Node artifacts
 node_modules
 

.pre-commit-config.yaml

@@ -3,15 +3,14 @@
 exclude: |
   (?x)(
     # Python/system files
-    ^.*/__init__\.py$|
     ^.*?/\.venv/.*$|
     ^.*?/node_modules/.*$|
-    
+
     # Generated/test files
     ^.*?/\.pytest_cache/.*$|
     ^.*?/__pycache__/.*$|
     ^.*?/\.mypy_cache/.*$|
-    ^.*?/\.ruff_cache/.*$
+    ^.*?/\.ruff_cache/.*$|
 
     # Package management
     ^.*?/poetry\.lock$|
@@ -22,7 +21,7 @@ exclude: |
     ^.*?/build/.*$|
     ^.*?/dist/.*$|
     ^.*?/\.coverage$|
-    ^.*?/coverage\.xml$|
+    ^.*?/coverage\.xml$
   )
 
 repos:
@@ -38,18 +37,15 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.11.5
     hooks:
-      # Run the linter with repo-defined settings
       - id: ruff
         args: [--fix]
-
-      # Run the formatter with repo-defined settings
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.0.3
     hooks:
       - id: prettier
-        types_or: [json, yaml]
+        args: [--write]
         additional_dependencies:
           - [email protected]
 
@@ -61,3 +57,20 @@ repos:
         language: golang
         additional_dependencies: [github.com/google/[email protected]]
         files: \.py$
+
+      - id: poetry-check
+        name: Check Poetry lockfile
+        entry: poetry check
+        language: system
+        pass_filenames: false
+        always_run: true
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        args: [--config-file=mypy.ini, --show-column-numbers]
+        files: ^airbyte_cdk/
+        pass_filenames: true

airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -39,6 +39,7 @@
 """
 
 import os
+import sys
 from pathlib import Path
 from types import ModuleType
 
@@ -161,15 +162,22 @@ def test(
         pytest_args.append("--collect-only")
 
     pytest_args.append(str(test_file_path))
+
+    test_results_dir = connector_directory / "build" / "test-results"
+    test_results_dir.mkdir(parents=True, exist_ok=True)
+    junit_xml_path = test_results_dir / "standard-tests-junit.xml"
+    pytest_args.extend(["--junitxml", str(junit_xml_path)])
+
     click.echo(f"Running tests from connector directory: {connector_directory}...")
     click.echo(f"Test file: {test_file_path}")
     click.echo(f"Collect only: {collect_only}")
     click.echo(f"Pytest args: {pytest_args}")
     click.echo("Invoking Pytest...")
-    pytest.main(
+    exit_code = pytest.main(
         pytest_args,
         plugins=[],
     )
+    sys.exit(exit_code)
 
 
 __all__ = [

airbyte_cdk/cli/airbyte_cdk/_secrets.py

@@ -49,7 +49,12 @@
     resolve_connector_name_and_directory,
 )
 
-AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing"
+GCP_PROJECT_ID: str = os.environ.get("GCP_PROJECT_ID", "") or "dataline-integration-testing"
+# We put the `or` outside the `get()` because we want the `GCP_PROJECT_ID`
+# env var to be ignored if it contains an empty string, such as in CI where the
+# workflow might set it to a value that is itself actually missing or unset.
+"""The GCP project ID to use for fetching integration test secrets."""
+
 CONNECTOR_LABEL = "connector"
 GLOBAL_MASK_KEYS_URL = "https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml"
 
@@ -83,8 +88,11 @@ def secrets_cli_group() -> None:
 @click.option(
     "--gcp-project-id",
     type=str,
-    default=AIRBYTE_INTERNAL_GCP_PROJECT,
-    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+    default=GCP_PROJECT_ID,
+    help=(
+        "GCP project ID for retrieving integration tests credentials. "
+        "Defaults to the value of the `GCP_PROJECT_ID` environment variable, if set."
+    ),
 )
 @click.option(
     "--print-ci-secrets-masks",
@@ -95,7 +103,7 @@ def secrets_cli_group() -> None:
 )
 def fetch(
     connector: str | Path | None = None,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    gcp_project_id: str = GCP_PROJECT_ID,
     print_ci_secrets_masks: bool = False,
 ) -> None:
     """Fetch secrets for a connector from Google Secret Manager.
@@ -192,41 +200,41 @@ def fetch(
 
 
 @secrets_cli_group.command("list")
-@click.option(
-    "--connector-name",
+@click.argument(
+    "connector",
+    required=False,
     type=str,
-    help="Name of the connector to fetch secrets for. Ignored if --connector-directory is provided.",
-)
-@click.option(
-    "--connector-directory",
-    type=click.Path(exists=True, file_okay=False, path_type=Path),
-    help="Path to the connector directory.",
+    metavar="[CONNECTOR]",
 )
 @click.option(
     "--gcp-project-id",
     type=str,
-    default=AIRBYTE_INTERNAL_GCP_PROJECT,
-    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+    default=GCP_PROJECT_ID,
+    help=(
+        "GCP project ID for retrieving integration tests credentials. "
+        "Defaults to the value of the `GCP_PROJECT_ID` environment variable, if set."
+    ),
 )
 def list_(
-    connector_name: str | None = None,
-    connector_directory: Path | None = None,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    connector: str | Path | None = None,
+    *,
+    gcp_project_id: str = GCP_PROJECT_ID,
 ) -> None:
     """List secrets for a connector from Google Secret Manager.
 
     This command fetches secrets for a connector from Google Secret Manager and prints
     them as a table.
 
+    [CONNECTOR] can be a connector name (e.g. 'source-pokeapi'), a path to a connector directory, or omitted to use the current working directory.
+    If a string containing '/' is provided, it is treated as a path. Otherwise, it is treated as a connector name.
+
     If no connector name or directory is provided, we will look within the current working
     directory. If the current working directory is not a connector directory (e.g. starting
     with 'source-') and no connector name or path is provided, the process will fail.
     """
     click.echo("Scanning secrets...", err=True)
 
-    connector_name = connector_name or resolve_connector_name(
-        connector_directory=connector_directory or Path().resolve().absolute(),
-    )
+    connector_name, _ = resolve_connector_name_and_directory(connector)
     secrets: list[Secret] = _fetch_secret_handles(  # type: ignore
         connector_name=connector_name,
         gcp_project_id=gcp_project_id,
@@ -303,7 +311,7 @@ def _get_secret_url(secret_name: str, gcp_project_id: str) -> str:
 
 def _fetch_secret_handles(
     connector_name: str,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    gcp_project_id: str = GCP_PROJECT_ID,
 ) -> list["Secret"]:  # type: ignore
     """Fetch secrets from Google Secret Manager."""
     if not secretmanager: