Merge remote-tracking branch 'origin/main' into baz/cdk/extract-common-manifest-parts

Author: Oleksandr Bazarnov

.github/workflows/connector-tests.yml

@@ -96,7 +96,7 @@ jobs:
     name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
     permissions:
       checks: write
-      contents: write  # Required for creating commit statuses
+      contents: write # Required for creating commit statuses
       pull-requests: read
     steps:
       - name: Abort if extra not changed (${{matrix.cdk_extra}})

.github/workflows/docker-build-check.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   docker-build-check:
-    name: SDM Docker Image Build  # Renamed job to be more descriptive
+    name: SDM Docker Image Build # Renamed job to be more descriptive
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout code

.github/workflows/pypi_publish.yml

@@ -42,7 +42,6 @@ jobs:
     name: Build Python Package
     runs-on: ubuntu-24.04
     steps:
-
       - name: Checkout CDK Repo
         uses: actions/checkout@v4
         with:

.github/workflows/python_dependency_analysis.yml

@@ -19,20 +19,20 @@ jobs:
     name: Dependency Analysis with Deptry
     runs-on: ubuntu-24.04
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-    - name: Set up Poetry
-      uses: Gr1N/setup-poetry@v9
-      with:
-        poetry-version: "2.0.1"
-    - name: Install dependencies
-      run: poetry install --all-extras
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Set up Poetry
+        uses: Gr1N/setup-poetry@v9
+        with:
+          poetry-version: "2.0.1"
+      - name: Install dependencies
+        run: poetry install --all-extras
 
-    # Job-specific step(s):
-    - name: Run Deptry
-      run: |
-        poetry run deptry .
+      # Job-specific step(s):
+      - name: Run Deptry
+        run: |
+          poetry run deptry .

.github/workflows/semantic_pr_check.yml

@@ -39,7 +39,7 @@ jobs:
             Build
             tests
             Tests
-            
+
       - name: Check for "do not merge" in PR title
         if: ${{ github.event.pull_request.draft == false }}
         uses: actions/github-script@v6

.github/workflows/slash_command_dispatch.yml

@@ -51,7 +51,7 @@ jobs:
           - \`/test\` - Runs the test suite
           - \`/poetry-lock\` - Re-locks dependencies and updates the poetry.lock file
           - \`/help\` - Shows this help message"
-          
+
           if [[ "${{ github.event.comment.body }}" == "/help" ]]; then
             echo "body=$HELP_TEXT" >> $GITHUB_OUTPUT
           else

.pre-commit-config.yaml

@@ -36,7 +36,7 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.3
+    rev: v0.11.5
     hooks:
       # Run the linter with repo-defined settings
       - id: ruff

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,7 +4,7 @@
 
 
 from dataclasses import asdict, dataclass, field
-from typing import Any, Dict, List, Mapping, Optional
+from typing import Any, ClassVar, Dict, List, Mapping, Optional
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -37,6 +37,8 @@
 
 @dataclass
 class TestLimits:
+    __test__: ClassVar[bool] = False  # Tell Pytest this is not a Pytest class, despite its name
+
     max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
     max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
     max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
@@ -53,18 +55,33 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
     max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
     return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
+def normalize_manifest(config: Mapping[str, Any]) -> bool:
+    """
+    Check if the manifest should be normalized.
+    :param config: The configuration to check
+    :return: True if the manifest should be normalized, False otherwise.
+    """
+    return config.get("__requires_normalization", False)
+
+def post_resolve_manifest(config: Mapping[str, Any]) -> bool:
+    """
+    Check if the manifest should be post-resolved.
+    :param config: The configuration to check
+    :return: True if the manifest should be post-resolved, False otherwise.
+    """
+    return config.get("__post_resolve_manifest", False)
 
 def create_source(
     config: Mapping[str, Any],
     limits: TestLimits,
-    post_resolve_manifest: Optional[bool] = False,
 ) -> ManifestDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
     return ManifestDeclarativeSource(
         config=config,
         emit_connector_builder_messages=True,
         source_config=manifest,
-        post_resolve_manifest=post_resolve_manifest,
+        normalize_manifest=normalize_manifest(config),
+        post_resolve_manifest=post_resolve_manifest(config),
         component_factory=ModelToComponentFactory(
             emit_connector_builder_messages=True,
             limit_pages_fetched_per_slice=limits.max_pages_per_slice,

airbyte_cdk/connector_builder/main.py

@@ -78,9 +78,9 @@ def handle_connector_builder_request(
     if command == "resolve_manifest":
         return resolve_manifest(source)
     elif command == "test_read":
-        assert (
-            catalog is not None
-        ), "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
+        assert catalog is not None, (
+            "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
+        )
         return read_stream(source, config, catalog, state, limits)
     elif command == "full_resolve_manifest":
         return full_resolve_manifest(source, limits)

airbyte_cdk/connector_builder/test_reader/reader.py

@@ -4,7 +4,7 @@
 
 
 import logging
-from typing import Any, Dict, Iterator, List, Mapping, Optional, Union
+from typing import Any, ClassVar, Dict, Iterator, List, Mapping, Optional, Union
 
 from airbyte_cdk.connector_builder.models import (
     AuxiliaryRequest,
@@ -66,6 +66,8 @@ class TestReader:
 
     """
 
+    __test__: ClassVar[bool] = False  # Tell Pytest this is not a Pytest class, despite its name
+
     logger = logging.getLogger("airbyte.connector-builder")
 
     def __init__(