ci: create unified cdk, sdm, and builder publish flow (#107)

aaronsteers · web-flow · commit e1100ef3214e · 2024-12-04T02:45:51.000Z
diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml
@@ -1,5 +1,6 @@
 # This flow publishes the Source-Declarative-Manifest (SDM)
 # connector to DockerHub as a Docker image.
+# TODO: Delete this workflow file once the unified publish flow is implemented and proven stable.
 
 name: Publish SDM Connector
 
diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml
@@ -1,27 +1,98 @@
 # This workflow builds the python package.
-# On release tags, it also publishes to PyPI and DockerHub.
-# If we rename the workflow file name, we have to also update the
-# Trusted Publisher settings on PyPI.
+
+# When from from a release tags or a workflow dispatch, it also publishes to PyPI and DockerHub along
+# with bumping the Connector Builder pinned version.
+# Note: We may want to rename this file at some point. However, if we rename the workflow file name,
+# we have to also update the Trusted Publisher settings on PyPI.
+
 name: Packaging and Publishing
 
 on:
   push:
+    tags:
+      - 'v*'
   workflow_dispatch:
     inputs:
       version:
-        description: "The version to publish, ie 1.0.0 or 1.0.0-dev1"
+        description: "Version. The version to publish, ie 1.0.0 or 1.0.0-dev1. In most cases, you can leave this blank. If run from a release tag (recommended), the version number will be inferred from the git tag."
+        required: false
+      publish_to_pypi:
+        description: "Publish to PyPI. If true, the workflow will publish to PyPI."
+        type: boolean
+        required: true
+        default: true
+      publish_to_dockerhub:
+        description: "Publish to DockerHub. If true, the workflow will publish the SDM connector to DockerHub."
+        type: boolean
         required: true
+        default: true
+      update_connector_builder:
+        description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder."
+        type: boolean
+        required: true
+        default: true
 
 jobs:
   build:
+    name: Build Python Package
     runs-on: ubuntu-latest
     steps:
+      - name: Detect Release Tag Version
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          DETECTED_VERSION=${{ github.ref_name }}
+          echo "Version ref set to '${DETECTED_VERSION}'"
+          # Remove the 'v' prefix if it exists
+          DETECTED_VERSION="${DETECTED_VERSION#v}"
+          echo "Setting version to '$DETECTED_VERSION'"
+          echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV
+
+      - name: Validate and set VERSION from git ref ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
+        id: set_version
+        run: |
+          INPUT_VERSION=${{ github.event.inputs.version }}
+          echo "Version input set to '${INPUT_VERSION}'"
+          # Exit with success if both detected and input versions are empty
+          if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then
+            echo "No version detected or input. Will publish to SHA tag instead."
+            echo 'VERSION=' >> $GITHUB_ENV
+            exit 0
+          fi
+          # Remove the 'v' prefix if it exists
+          INPUT_VERSION="${INPUT_VERSION#v}"
+          # Fail if detected version is non-empty and different from the input version
+          if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
+            echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
+            exit 1
+          fi
+          # Set the version to the input version if non-empty, otherwise the detected version
+          VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
+          # Fail if the version is still empty
+          if [ -z "$VERSION" ]; then
+            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
+            exit 1
+          fi
+          echo "Setting version to '$VERSION'"
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
+          # Check if version is a prerelease version (will not tag 'latest')
+          if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "IS_PRERELEASE=false" >> $GITHUB_ENV
+            echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT
+          else
+            echo "IS_PRERELEASE=true" >> $GITHUB_ENV
+            echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
+          fi
+
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{ github.event_name == 'workflow_dispatch' && format('v{0}', github.event.inputs.version) || github.ref }}
 
       - uses: hynek/build-and-inspect-python-package@v2
+        env:
+          # Pass in the evaluated version from the previous step
+          # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables
+          POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}}
 
       - uses: actions/upload-artifact@v4
         with:
@@ -30,7 +101,11 @@ jobs:
             /tmp/baipp/dist/*.whl
             /tmp/baipp/dist/*.tar.gz
 
-  publish:
+    outputs:
+      VERSION: ${{ steps.set_version.outputs.VERSION }}
+      IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }}
+
+  publish_cdk:
     name: Publish CDK version to PyPI
     runs-on: ubuntu-latest
     needs: [build]
@@ -39,8 +114,11 @@ jobs:
       contents: write
     environment:
       name: PyPi
-      url: https://pypi.org/p/airbyte
-    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
+      url: https://pypi.org/p/airbyte-cdk/
+    if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && (github.event.inputs.publish_to_pypi == 'true' || github.event.inputs.update_connector_builder == 'true'))
+    env:
+      VERSION: ${{ needs.build.outputs.VERSION }}
+      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -58,4 +136,217 @@ jobs:
           file_glob: true
 
       - name: Publish to PyPI
+        if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || github.event.inputs.publish_to_pypi == 'true'
         uses: pypa/gh-action-pypi-publish@v1.10.3
+
+  publish_sdm:
+    name: Publish SDM to DockerHub
+    # TODO: When we're ready to publish after each release, prefix the `if` below with:
+    #         (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) ||
+    # Until then, this workflow needs to be kicked off manually.
+    # Last remaining blocker documented here: https://github.com/airbytehq/airbyte-python-cdk/issues/64
+    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_dockerhub == 'true')
+    runs-on: ubuntu-latest
+    needs: [build]
+    environment:
+      name: DockerHub
+      url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
+    env:
+      VERSION: ${{ needs.build.outputs.VERSION }}
+      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # We need to download the build artifact again because the previous job was on a different runner
+      - name: Download Build Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages-${{ github.run_id }}
+          path: dist
+
+      - name: Set up QEMU for multi-platform builds
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
+        if: env.VERSION != ''
+        run: |
+          tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
+          if [ -z "$tag" ]; then
+            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
+            exit 1
+          fi
+          echo "Checking if tag '$tag' exists on DockerHub..."
+          if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
+            echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
+            exit 1
+          fi
+          echo "No existing tag '$tag' found. Proceeding with publish."
+
+      - name: "Build and push (sha tag: '${{ github.sha }}')"
+        # Only run if the version is not set
+        if: env.VERSION == ''
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/source-declarative-manifest:${{ github.sha }}
+
+      - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
+        # Only run if the version is set
+        if: env.VERSION != ''
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/source-declarative-manifest:${{ env.VERSION }}
+
+      - name: Build and push ('latest' tag)
+        # Only run if version is set and IS_PRERELEASE is false
+        if: env.VERSION != '' && env.IS_PRERELEASE == 'false'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/source-declarative-manifest:latest
+
+  update-connector-builder:
+    # Create a PR against the Builder, to update the CDK version that it uses.
+    # In the future, Builder may use the SDM docker image instead of the Python CDK package.
+    name: Bump Connector Builder CDK version
+    needs:
+      - build
+      - publish_cdk
+    if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true')
+    env:
+      VERSION: ${{ needs.build.outputs.VERSION }}
+      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Checkout Airbyte Platform Internal
+        uses: actions/checkout@v4
+        with:
+          repository: airbytehq/airbyte-platform-internal
+          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+      - name: Update Builder's CDK version to ${{ env.VERSION }}
+        # PyPI servers aren't immediately updated so we may need to retry a few times.
+        uses: nick-fields/retry@v3
+        with:
+          shell: bash
+          max_attempts: 5
+          retry_wait_seconds: 30
+          timeout_minutes: 7
+          command: |
+            set -euo pipefail
+            PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
+            sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile"
+            sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "cloud/airbyte-connector-builder-server-wrapped/Dockerfile"
+            sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in
+            echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION
+            cd oss/airbyte-connector-builder-server
+            python -m pip install --no-cache-dir pip-tools
+            pip-compile --upgrade
+      - name: Create Pull Request
+        id: create-pull-request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          commit-message: "chore: update CDK version following release"
+          title: "chore: update CDK version following release"
+          body: This is an automatically generated PR triggered by a CDK release
+          branch: automatic-cdk-release
+          base: master
+          delete-branch: true
+      - name: Post success to Slack channel dev-connectors-extensibility
+        uses: slackapi/slack-github-action@v1.23.0
+        continue-on-error: true
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          channel-id: C04J1M66D8B
+          # Channel: #dev-connectors-extensibility-releases
+          # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B
+          payload: |
+            {
+                "text": "A new version of Python CDK has been released!",
+                "blocks": [
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "Python CDK `v${{ env.VERSION }}` has been <https://github.com/airbytehq/airbyte-python-cdk/releases|released>\n\n"
+                        }
+                    },
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n"
+                        }
+                    },
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
+                        }
+                    }
+                ]
+            }
+      - name: Post failure to Slack channel dev-connectors-extensibility
+        if: failure()
+        uses: slackapi/slack-github-action@v1.23.0
+        continue-on-error: true
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          channel-id: C04J1M66D8B
+          # Channel: #dev-connectors-extensibility-releases
+          # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B
+          payload: |
+            {
+                "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated",
+                "blocks": [
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "Python CDK `v${{ env.VERSION }}` has been <https://github.com/airbytehq/airbyte-python-cdk/releases|released>\n\n"
+                        }
+                    },
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": ":warning: Could not automatically create a PR for Connector Builder>\n"
+                        }
+                    },
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
+                        }
+                    }
+                ]
+            }
diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml
@@ -7,6 +7,25 @@ on:
   pull_request:
 
 jobs:
+  test-build:
+    name: Build and Inspect Python Package
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - uses: hynek/build-and-inspect-python-package@v2
+        env:
+          # Pass in dummy version '0.0.0dev0' version to appease dynamic versioning
+          POETRY_DYNAMIC_VERSIONING_BYPASS: 0.0.0dev0
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Packages-${{ github.run_id }}
+          path: |
+            /tmp/baipp/dist/*.whl
+            /tmp/baipp/dist/*.tar.gz
+
   pytest-fast:
     name: Pytest (Fast)
     runs-on: ubuntu-latest
