diff --git a/.github/workflows/connectors_up_to_date.yml b/.github/workflows/connectors_up_to_date.yml index 29a051c21548..198b7345573f 100644 --- a/.github/workflows/connectors_up_to_date.yml +++ b/.github/workflows/connectors_up_to_date.yml @@ -10,8 +10,8 @@ on: workflow_dispatch: inputs: connectors-options: - description: "Options to pass to the 'airbyte-ci connectors' command group." - default: "--language=python --language=low-code --language=manifest-only" + description: "Extra flags to pass to `airbyte-ops local connector list` (e.g. --name=source-github)." + default: "" jobs: generate_matrix: name: Generate matrix @@ -21,29 +21,58 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - name: Run airbyte-ci connectors list [SCHEDULED TRIGGER] - if: github.event_name == 'schedule' - id: airbyte-ci-connectors-list-scheduled - uses: ./.github/actions/run-airbyte-ci - with: - context: "master" - subcommand: 'connectors --language=python --language=low-code --language=manifest-only --metadata-query="\"-rc.\" not in data.dockerImageTag and \"source-declarative-manifest\" not in data.dockerRepository" list --output=selected_connectors.json' - - name: Run airbyte-ci connectors list [MANUAL TRIGGER] - if: github.event_name == 'workflow_dispatch' - id: airbyte-ci-connectors-list-workflow-dispatch - uses: ./.github/actions/run-airbyte-ci - with: - context: "master" - subcommand: 'connectors ${{ github.event.inputs.connectors-options }} --metadata-query="\"-rc.\" not in data.dockerImageTag and \"source-declarative-manifest\" not in data.dockerRepository" list --output=selected_connectors.json' - # We generate a dynamic matrix from the list of selected connectors. - # A matrix is required in this situation because the number of connectors is large and running them all in a single job would exceed the maximum job time limit of 6 hours. - # We use 25 connectors per job, with hopes they can finish within the 1 hours duration of the GitHub App's - # token. + + - name: Install uv + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 + + - name: Install airbyte-ops CLI + run: uv tool install airbyte-internal-ops + + # --- Connector selection (three-call pattern) --- + # Call 1: Sources - all support levels, Python/low-code/manifest-only languages + - name: List source connectors + id: list-sources + env: + CONNECTOR_OPTIONS: ${{ github.event.inputs.connectors-options }} + run: | + SOURCES=$( + airbyte-ops local connector list \ + --repo-path "$GITHUB_WORKSPACE" \ + --connector-type=source \ + --language python --language low-code --language manifest-only \ + --exclude-connectors=source-declarative-manifest \ + $CONNECTOR_OPTIONS \ + --output-format=csv + ) + echo "sources=$SOURCES" | tee -a $GITHUB_OUTPUT + + # Call 2: Destinations - certified only, Python/low-code/manifest-only languages + - name: List destination connectors + id: list-destinations + env: + CONNECTOR_OPTIONS: ${{ github.event.inputs.connectors-options }} + run: | + DESTINATIONS=$( + airbyte-ops local connector list \ + --repo-path "$GITHUB_WORKSPACE" \ + --connector-type=destination \ + --certified-only \ + --language python --language low-code --language manifest-only \ + $CONNECTOR_OPTIONS \ + --output-format=csv + ) + echo "destinations=$DESTINATIONS" | tee -a $GITHUB_OUTPUT + + # Call 3: Combine both sets and output as GH Actions matrix - name: Generate matrix - 25 connectors per job id: generate_matrix run: | - matrix=$(jq -c -r '{include: [.[] | "--name=" + .] | to_entries | group_by(.key / 25 | floor) | map(map(.value) | {"connector_names": join(" ")})}' selected_connectors.json) - echo "generated_matrix=$matrix" >> $GITHUB_OUTPUT + echo "generated_matrix=$( \ + airbyte-ops local connector list \ + --repo-path "$GITHUB_WORKSPACE" \ + --connectors-filter="${{ steps.list-sources.outputs.sources }},${{ steps.list-destinations.outputs.destinations }}" \ + --output-format=json-gh-matrix \ + )" | tee -a $GITHUB_OUTPUT run_connectors_up_to_date: needs: generate_matrix @@ -51,12 +80,24 @@ jobs: runs-on: connector-up-to-date-medium continue-on-error: true strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.generated_matrix)}} + matrix: ${{ fromJson(needs.generate_matrix.outputs.generated_matrix) }} permissions: pull-requests: write + contents: write + env: + CONNECTOR_NAME: ${{ matrix.connector }} + CONNECTOR_DIR: ${{ matrix.connector_dir }} + CONNECTOR_LANGUAGE: ${{ matrix.connector_language }} steps: - name: Checkout Airbyte uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Install uv + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 + + - name: Install airbyte-ops CLI + run: uv tool install airbyte-internal-ops + - name: Authenticate as 'octavia-bot-hoard' GitHub App uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 id: get-app-token @@ -65,19 +106,121 @@ jobs: repositories: ${{ inputs.repo || github.event.repository.name }} app-id: ${{ secrets.OCTAVIA_BOT_HOARD_APP_ID }} private-key: ${{ secrets.OCTAVIA_BOT_HOARD_PRIVATE_KEY }} - # Token is good for 1 hour - - name: Run airbyte-ci connectors up-to-date [WORKFLOW] - id: airbyte-ci-connectors-up-to-date-workflow-dispatch - uses: ./.github/actions/run-airbyte-ci + + - name: Get GitHub App User ID + id: get-user-id + env: + GH_TOKEN: ${{ steps.get-app-token.outputs.token }} + run: echo "user-id=$(gh api "/users/${{ steps.get-app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" + + - name: Configure git author + run: | + git config --global user.name '${{ steps.get-app-token.outputs.app-slug }}[bot]' + git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.get-app-token.outputs.app-slug }}[bot]@users.noreply.github.com' + + - name: Docker login + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: - context: "master" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }} - docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} - docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} - gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} - gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - github_token: ${{ steps.get-app-token.outputs.token }} - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - subcommand: "connectors --concurrency=10 ${{ matrix.connector_names}} up-to-date --create-prs --auto-merge" + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + # --- Step 5a: Create or update a PR (first pass) --- + # We create the PR early so we have a PR number for the changelog entry. + # peter-evans/create-pull-request handles branch creation, committing, and PR upsert. + # On subsequent runs for the same connector, it updates the existing PR. + - name: Create or update PR (initial) + id: create-pr + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 + with: + token: ${{ steps.get-app-token.outputs.token }} + commit-message: "chore(${{ env.CONNECTOR_NAME }}): update dependencies" + branch: "connectors-up-to-date/${{ env.CONNECTOR_NAME }}" + delete-branch: true + title: "chore(${{ env.CONNECTOR_NAME }}): update dependencies" + body: | + Updates dependencies for `${{ env.CONNECTOR_NAME }}`. + + 🤖 Generated by [automated workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). + labels: | + area/connectors + auto-merge + draft: true + + # --- Step 2: Run `poetry update` (Python connectors only) --- + # TK-TODO(https://github.com/airbytehq/airbyte-ops-mcp/issues/628): + # Replace inline poetry steps with `airbyte-ops local connector bump-external-dependencies` + # once the CLI command exists. For now, poetry update is handled directly below. + - name: Set up Python + if: env.CONNECTOR_LANGUAGE == 'python' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Install Poetry + if: env.CONNECTOR_LANGUAGE == 'python' + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1 + with: + version: 1.8.5 + + - name: Run poetry update + if: env.CONNECTOR_LANGUAGE == 'python' + working-directory: ${{ env.CONNECTOR_DIR }} + run: poetry update --lock + + # TK-TODO(https://github.com/airbytehq/airbyte-ops-mcp/issues/627): + # Add a step for `airbyte-ops local connector bump-cdk` once the CLI command exists. + # This should bump the CDK dependency independently of other updates. + # A workflow input (`bump-cdk`: true/false/latest) should gate this step. + + # --- Steps 3 & 4: Bump version + changelog entry --- + - name: Bump connector version (patch) with changelog + env: + PR_NUMBER: ${{ steps.create-pr.outputs.pull-request-number }} + run: | + airbyte-ops local connector bump-version \ + --name "$CONNECTOR_NAME" \ + --repo-path "$GITHUB_WORKSPACE" \ + --bump-type patch \ + --changelog-message "Update dependencies" \ + --pr-number "$PR_NUMBER" + + # --- Step 1: Update base image (TK-TODO) --- + # TK-TODO(https://github.com/airbytehq/airbyte-ops-mcp/issues/626): + # `airbyte-ops local connector bump-base-image` does not exist yet. + # This step should update the base image tag in metadata.yaml to the latest stable version. + # Placed near the end of the process so it picks up the bumped version. + - name: Update base image in metadata.yaml + run: | + airbyte-ops local connector bump-base-image \ + --name "$CONNECTOR_NAME" \ + --repo-path "$GITHUB_WORKSPACE" + + # --- Step 5b: Push all changes to the PR branch --- + - name: Push changes to PR branch + env: + PR_BRANCH: ${{ steps.create-pr.outputs.pull-request-branch }} + GH_TOKEN: ${{ steps.get-app-token.outputs.token }} + run: | + git add "$CONNECTOR_DIR" + if git diff --cached --quiet; then + echo "No changes to commit." + else + git commit -m "chore($CONNECTOR_NAME): update dependencies" + git push origin "HEAD:$PR_BRANCH" + fi + + # Mark the PR as ready for review now that all changes are pushed. + - name: Mark PR as ready for review + if: steps.create-pr.outputs.pull-request-number + env: + GH_TOKEN: ${{ steps.get-app-token.outputs.token }} + PR_NUMBER: ${{ steps.create-pr.outputs.pull-request-number }} + run: gh pr ready "$PR_NUMBER" + + # --- Step 6: Set auto-merge label + enable native auto-merge --- + - name: Enable auto-merge on PR + if: steps.create-pr.outputs.pull-request-number + env: + GH_TOKEN: ${{ steps.get-app-token.outputs.token }} + PR_NUMBER: ${{ steps.create-pr.outputs.pull-request-number }} + run: gh pr merge "$PR_NUMBER" --squash --auto