Merge branch 'main' into dbgold17/refactor-cdk-oauth

Author: dbgold17

.github/workflows/connector-tests.yml

@@ -25,7 +25,7 @@ concurrency:
 jobs:
   cdk_changes:
     name: Get Changes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       statuses: write
       pull-requests: read
@@ -62,7 +62,7 @@ jobs:
     # Forked PRs are handled by the community_ci.yml workflow
     # If the condition is not met the job will be skipped (it will not fail)
     # runs-on: connector-test-large
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 360 # 6 hours
     strategy:
       fail-fast: false
@@ -75,11 +75,11 @@ jobs:
           # Chargebee is being flaky:
           # - connector: source-chargebee
           #   cdk_extra: n/a
-          # These two are behind in CDK updates and can't be used as tests until they are updated:
-          # - connector: source-s3
-          #   cdk_extra: file-based
+          # This one is behind in CDK updates and can't be used as tests until it is updated:
           # - connector: destination-pinecone
           #   cdk_extra: vector-db-based
+          - connector: source-google-drive
+            cdk_extra: file-based
           - connector: destination-motherduck
             cdk_extra: sql
           # ZenDesk currently failing (as of 2024-12-02)
@@ -96,7 +96,7 @@ jobs:
     name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
     permissions:
       checks: write
-      contents: write  # Required for creating commit statuses
+      contents: write # Required for creating commit statuses
       pull-requests: read
     steps:
       - name: Abort if extra not changed (${{matrix.cdk_extra}})
@@ -153,7 +153,7 @@ jobs:
           POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
         run: |
           cd airbyte
-          make tools.airbyte-ci-dev.install
+          make tools.airbyte-ci.install
           airbyte-ci \
           --ci-report-bucket-name=airbyte-ci-reports-multi \
           connectors \

.github/workflows/docker-build-check.yml

@@ -0,0 +1,44 @@
+name: Docker Build Check
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  docker-build-check:
+    name: SDM Docker Image Build # Renamed job to be more descriptive
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # Build the Python package to create the wheel files needed for the Docker build
+      - name: Build Python Package
+        uses: hynek/build-and-inspect-python-package@v2
+        env:
+          POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0dev0"
+
+      # Copy the wheel files to the dist directory
+      - name: Copy wheel files to dist directory
+        run: |
+          mkdir -p dist
+          cp /tmp/baipp/dist/*.whl dist/
+
+      - name: Set up QEMU for multi-platform builds
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image for multiple platforms
+        id: docker-build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: false
+          tags: airbyte/source-declarative-manifest:pr-${{ github.event.pull_request.number }}
+          outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=SDM Docker image for PR ${{ github.event.pull_request.number }}

.github/workflows/poe-command.yml

@@ -0,0 +1,28 @@
+name: On-Demand Poe Task
+
+on:
+  workflow_dispatch:
+    inputs:
+      comment-id:
+        description: "Optional comment-id of the slash command. Ignore if not applicable."
+        required: false
+      pr:
+        description: "PR Number"
+        required: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  run-poe-command:
+    env:
+      GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run Poe Slash Command Processor
+        uses: aaronsteers/poe-command-processor@v1
+        with:
+          pr: ${{ github.event.inputs.pr }}
+          comment-id: ${{ github.event.inputs.comment-id }}
+          github-token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}

.github/workflows/pypi_publish.yml

@@ -5,7 +5,7 @@
 # Note: We may want to rename this file at some point. However, if we rename the workflow file name,
 # we have to also update the Trusted Publisher settings on PyPI.
 
-name: Packaging and Publishing
+name: CDK Publish
 
 on:
   push:
@@ -14,7 +14,12 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: "Note that this workflow is intended for prereleases. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow from main or from a dev branch, please enter the desired version number here, for instance 1.2.3dev0 or 1.2.3rc1."
+        description: >
+          Note that this workflow is intended for prereleases. For public-facing stable releases,
+          please use the GitHub Releases workflow instead:
+          https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md.
+          For prereleases, please leave the version blank to use the detected version. Alternatively,
+          you can override the dynamic versioning for special use cases.
         required: false
       publish_to_pypi:
         description: "Publish to PyPI. If true, the workflow will publish to PyPI."
@@ -37,17 +42,29 @@ jobs:
     name: Build Python Package
     runs-on: ubuntu-24.04
     steps:
-      - name: Detect Release Tag Version
+      - name: Checkout CDK Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect Prerelease Version using Dunamai
+        uses: mtkennerly/dunamai-action@v1
+        with:
+          args: --format "{base}.post{distance}.dev${{ github.run_id }}"
+          env-var: DETECTED_VERSION
+
+      - name: Detect Release Tag Version from git ref ('${{ github.ref_name }}')
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
+          echo "Overriding Dunamai detected version: '${{ env.DETECTED_VERSION || 'none' }}'"
+          # Extract the version from the git ref
           DETECTED_VERSION=${{ github.ref_name }}
-          echo "Version ref set to '${DETECTED_VERSION}'"
           # Remove the 'v' prefix if it exists
           DETECTED_VERSION="${DETECTED_VERSION#v}"
-          echo "Setting version to '$DETECTED_VERSION'"
+          echo "Setting detected version to '$DETECTED_VERSION'"
           echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV
 
-      - name: Validate and set VERSION from git ref ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
+      - name: Validate and set VERSION (detected='${{ env.DETECTED_VERSION }}', input='${{ github.event.inputs.version || 'none' }}')
         id: set_version
         run: |
           INPUT_VERSION=${{ github.event.inputs.version }}
@@ -62,8 +79,8 @@ jobs:
           INPUT_VERSION="${INPUT_VERSION#v}"
           # Fail if detected version is non-empty and different from the input version
           if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
-            echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
-            exit 1
+            echo "Warning: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
+            echo "Using input version '${INPUT_VERSION}' instead."
           fi
           # Set the version to the input version if non-empty, otherwise the detected version
           VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
@@ -84,10 +101,6 @@ jobs:
             echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
           fi
 
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
       - uses: hynek/build-and-inspect-python-package@v2
         env:
           # Pass in the evaluated version from the previous step

.github/workflows/python_dependency_analysis.yml

@@ -0,0 +1,38 @@
+name: Dependency Analysis
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "airbyte_cdk/**"
+      - "poetry.lock"
+      - "pyproject.toml"
+  pull_request:
+    paths:
+      - "airbyte_cdk/**"
+      - "poetry.lock"
+      - "pyproject.toml"
+
+jobs:
+  dependency-analysis:
+    name: Dependency Analysis with Deptry
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Set up Poetry
+        uses: Gr1N/setup-poetry@v9
+        with:
+          poetry-version: "2.0.1"
+      - name: Install dependencies
+        run: poetry install --all-extras
+
+      # Job-specific step(s):
+      - name: Run Deptry
+        run: |
+          poetry run deptry .

.github/workflows/semantic_pr_check.yml

@@ -39,3 +39,13 @@ jobs:
             Build
             tests
             Tests
+
+      - name: Check for "do not merge" in PR title
+        if: ${{ github.event.pull_request.draft == false }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const title = context.payload.pull_request.title.toLowerCase();
+            if (title.includes('do not merge') || title.includes('do-not-merge')) {
+              core.setFailed('PR title contains "do not merge" or "do-not-merge". Please remove this before merging.');
+            }

.github/workflows/slash_command_dispatch.yml

@@ -6,8 +6,6 @@ on:
 
 jobs:
   slashCommandDispatch:
-    # Only allow slash commands on pull request (not on issues)
-    if: ${{ github.event.issue.pull_request }}
     runs-on: ubuntu-24.04
     steps:
       - name: Slash Command Dispatch
@@ -19,17 +17,25 @@ jobs:
           repository: ${{ github.repository }}
           token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           dispatch-type: workflow
-          issue-type: pull-request
+          issue-type: both
+
+          # Only run for users with 'write' permission on the main repository
+          permission: write
+
           commands: |
             autofix
             test
             poetry-lock
+            poe
+
+          # Notes regarding static-args:
+          # - Slash commands can be invoked from both issues and comments.
+          # - If the slash command is invoked from an issue, we intentionally pass 'null' as the PR number.
+          # - Comment ID will always be sent, and this is sufficient to post back status updates to the originating comment.
           static-args: |
-            pr=${{ github.event.issue.number }}
+            pr=${{ github.event.issue.pull_request != null && github.event.issue.number || '' }}
             comment-id=${{ github.event.comment.id }}
 
-          # Only run for users with 'write' permission on the main repository
-          permission: write
 
       - name: Edit comment with error message
         if: steps.dispatch.outputs.error-message
@@ -51,7 +57,7 @@ jobs:
           - \`/test\` - Runs the test suite
           - \`/poetry-lock\` - Re-locks dependencies and updates the poetry.lock file
           - \`/help\` - Shows this help message"
-          
+
           if [[ "${{ github.event.comment.body }}" == "/help" ]]; then
             echo "body=$HELP_TEXT" >> $GITHUB_OUTPUT
           else

.gitignore

@@ -15,3 +15,4 @@ dist
 .idea
 .vscode
 **/__pycache__
+.tmp

.pre-commit-config.yaml

@@ -36,7 +36,7 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.3
+    rev: v0.11.5
     hooks:
       # Run the linter with repo-defined settings
       - id: ruff

Dockerfile

@@ -5,7 +5,7 @@
 # A new version of source-declarative-manifest is built for every new Airbyte CDK release, and their versions are kept in sync.
 #
 
-FROM docker.io/airbyte/python-connector-base:3.0.0@sha256:1a0845ff2b30eafa793c6eee4e8f4283c2e52e1bbd44eed6cb9e9abd5d34d844
+FROM docker.io/airbyte/python-connector-base:4.0.0@sha256:d9894b6895923b379f3006fa251147806919c62b7d9021b5cd125bb67d7bbe22
 
 WORKDIR /airbyte/integration_code
 
@@ -24,8 +24,8 @@ RUN pip install dist/*.whl
 RUN mkdir -p source_declarative_manifest \
     && echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n    run()' > main.py \
     && touch source_declarative_manifest/__init__.py \
-    && cp /usr/local/lib/python3.10/site-packages/airbyte_cdk/cli/source_declarative_manifest/_run.py source_declarative_manifest/run.py \
-    && cp /usr/local/lib/python3.10/site-packages/airbyte_cdk/cli/source_declarative_manifest/spec.json source_declarative_manifest/
+    && cp /usr/local/lib/python3.11/site-packages/airbyte_cdk/cli/source_declarative_manifest/_run.py source_declarative_manifest/run.py \
+    && cp /usr/local/lib/python3.11/site-packages/airbyte_cdk/cli/source_declarative_manifest/spec.json source_declarative_manifest/
 
 # Remove unnecessary build files
 RUN rm -rf dist/ pyproject.toml poetry.lock README.md