Merge branch 'main' into feat-49796-azure_virtual_machines_operator

Author: nailo2c

.asf.yaml

@@ -171,6 +171,7 @@ github:
     - gyli
     - jroachgolf84
     - Dev-iL
+    - kacpermuda
 
 notifications:
   jobs: jobs@airflow.apache.org

.codespellignorelines

@@ -4,3 +4,4 @@
     The platform supports **C**reate, **R**ead, **U**pdate, and **D**elete operations on most resources.
 <pre><code>Code block\ndoes not\nrespect\nnewlines\n</code></pre>
       "trough",
+                assert "task_instance_id" in route.dependant.path_param_names, (

.github/workflows/registry-backfill.yml

@@ -0,0 +1,266 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Registry Backfill
+on:  # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      destination:
+        description: >
+          Publish to live or staging S3 bucket
+        required: true
+        type: choice
+        options:
+          - staging
+          - live
+        default: staging
+      providers:
+        description: >
+          Space-separated provider IDs
+          (e.g. 'amazon google databricks')
+        required: true
+        type: string
+      versions:
+        description: >
+          Space-separated versions to backfill
+          (e.g. '9.15.0 9.14.0'). Applied to ALL providers.
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+      bucket: ${{ steps.destination.outputs.bucket }}
+    steps:
+      - name: "Build provider matrix"
+        id: matrix
+        env:
+          PROVIDERS: ${{ inputs.providers }}
+        run: |
+          MATRIX=$(echo "${PROVIDERS}" \
+            | tr ' ' '\n' | jq -R . \
+            | jq -cs '{"provider": .}')
+          echo "matrix=${MATRIX}" >> "${GITHUB_OUTPUT}"
+
+      - name: "Determine S3 destination"
+        id: destination
+        env:
+          DESTINATION: ${{ inputs.destination }}
+        run: |
+          if [[ "${DESTINATION}" == "live" ]]; then
+            URL="s3://live-docs-airflow-apache-org"
+          else
+            URL="s3://staging-docs-airflow-apache-org"
+          fi
+          echo "bucket=${URL}/registry/" \
+            >> "${GITHUB_OUTPUT}"
+
+  backfill:
+    needs: prepare
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}
+    name: "Backfill ${{ matrix.provider }}"
+    if: >
+      contains(fromJSON('[
+        "ashb",
+        "bugraoz93",
+        "eladkal",
+        "ephraimbuddy",
+        "jedcunningham",
+        "jscheffl",
+        "kaxil",
+        "pierrejeambrun",
+        "shahar1",
+        "potiuk",
+        "utkarsharma2",
+        "vincbeck"
+        ]'), github.event.sender.login)
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: "Fetch provider tags"
+        env:
+          VERSIONS: ${{ inputs.versions }}
+          PROVIDER: ${{ matrix.provider }}
+        run: |
+          for VERSION in ${VERSIONS}; do
+            TAG="providers-${PROVIDER}/${VERSION}"
+            echo "Fetching tag: ${TAG}"
+            git fetch origin tag "${TAG}" \
+              2>/dev/null || echo "Tag not found"
+          done
+
+      - name: "Install uv"
+        uses: astral-sh/setup-uv@bd01e18f51369d5765a7df3681d34498e332e27e  # v6.3.1
+
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+        with:
+          python-version: "3.12"
+
+      - name: "Install AWS CLI v2"
+        run: |
+          curl -sSf \
+            "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \
+            -o /tmp/awscliv2.zip
+          unzip -q /tmp/awscliv2.zip -d /tmp
+          rm /tmp/awscliv2.zip
+          sudo /tmp/aws/install --update
+          rm -rf /tmp/aws/
+
+      - name: "Configure AWS credentials"
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7  # v6.0.0
+        with:
+          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - name: "Download existing providers.json"
+        env:
+          S3_BUCKET: ${{ needs.prepare.outputs.bucket }}
+        run: |
+          aws s3 cp \
+            "${S3_BUCKET}api/providers.json" \
+            dev/registry/providers.json || true
+
+      - name: "Extract version metadata from git tags"
+        env:
+          VERSIONS: ${{ inputs.versions }}
+          PROVIDER: ${{ matrix.provider }}
+        run: |
+          VERSION_ARGS=""
+          for VERSION in ${VERSIONS}; do
+            VERSION_ARGS="${VERSION_ARGS} --version ${VERSION}"
+          done
+          uv run python dev/registry/extract_versions.py \
+            --provider "${PROVIDER}" ${VERSION_ARGS} || true
+
+      - name: "Run breeze registry backfill"
+        env:
+          VERSIONS: ${{ inputs.versions }}
+          PROVIDER: ${{ matrix.provider }}
+        run: |
+          VERSION_ARGS=""
+          for VERSION in ${VERSIONS}; do
+            VERSION_ARGS="${VERSION_ARGS} --version ${VERSION}"
+          done
+          breeze registry backfill \
+            --provider "${PROVIDER}" ${VERSION_ARGS}
+
+      - name: "Download data files from S3 for build"
+        env:
+          S3_BUCKET: ${{ needs.prepare.outputs.bucket }}
+        run: |
+          aws s3 cp \
+            "${S3_BUCKET}api/providers.json" \
+            registry/src/_data/providers.json
+          aws s3 cp \
+            "${S3_BUCKET}api/modules.json" \
+            registry/src/_data/modules.json
+
+      - name: "Setup pnpm"
+        uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061  # v4.2.0
+        with:
+          version: 9
+
+      - name: "Setup Node.js"
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f  # v6.3.0
+        with:
+          node-version: 20
+          cache: 'pnpm'
+          cache-dependency-path: 'registry/pnpm-lock.yaml'
+
+      - name: "Install Node.js dependencies"
+        working-directory: registry
+        run: pnpm install --frozen-lockfile
+
+      - name: "Build registry site"
+        working-directory: registry
+        env:
+          REGISTRY_PATH_PREFIX: "/registry/"
+        run: pnpm build
+
+      - name: "Sync backfilled version pages to S3"
+        env:
+          S3_BUCKET: ${{ needs.prepare.outputs.bucket }}
+          CACHE_CONTROL: "public, max-age=300"
+          VERSIONS: ${{ inputs.versions }}
+          PROVIDER: ${{ matrix.provider }}
+        run: |
+          for VERSION in ${VERSIONS}; do
+            echo "Syncing ${PROVIDER}/${VERSION}..."
+            aws s3 sync \
+              "registry/_site/providers/${PROVIDER}/${VERSION}/" \
+              "${S3_BUCKET}providers/${PROVIDER}/${VERSION}/" \
+              --cache-control "${CACHE_CONTROL}"
+            aws s3 sync \
+              "registry/_site/api/providers/${PROVIDER}/${VERSION}/" \
+              "${S3_BUCKET}api/providers/${PROVIDER}/${VERSION}/" \
+              --cache-control "${CACHE_CONTROL}"
+          done
+
+  publish-versions:
+    needs: [prepare, backfill]
+    runs-on: ubuntu-latest
+    name: "Publish versions.json"
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+        with:
+          python-version: "3.12"
+
+      - name: "Install AWS CLI v2"
+        run: |
+          curl -sSf \
+            "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \
+            -o /tmp/awscliv2.zip
+          unzip -q /tmp/awscliv2.zip -d /tmp
+          rm /tmp/awscliv2.zip
+          sudo /tmp/aws/install --update
+          rm -rf /tmp/aws/
+
+      - name: "Configure AWS credentials"
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7  # v6.0.0
+        with:
+          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - name: "Publish version metadata"
+        env:
+          S3_BUCKET: ${{ needs.prepare.outputs.bucket }}
+        run: >
+          breeze registry publish-versions
+          --s3-bucket "${S3_BUCKET}"

.github/workflows/registry-build.yml

@@ -239,7 +239,14 @@ jobs:
           S3_BUCKET: ${{ steps.destination.outputs.bucket }}
         run: |
           aws s3 sync registry/_site/ "${S3_BUCKET}" \
-            --cache-control "${REGISTRY_CACHE_CONTROL}"
+            --cache-control "${REGISTRY_CACHE_CONTROL}" \
+            --exclude "pagefind/*"
+          # Pagefind generates content-hashed filenames (e.g. en_181da6f.pf_index).
+          # Each rebuild produces new hashes, so --delete is needed to remove stale
+          # index files. This is separate from the main sync which intentionally
+          # omits --delete to preserve files written by other steps (publish-versions).
+          aws s3 sync registry/_site/pagefind/ "${S3_BUCKET}pagefind/" \
+            --cache-control "${REGISTRY_CACHE_CONTROL}" --delete
 
       - name: "Publish version metadata"
         env:

Dockerfile

@@ -1201,7 +1201,7 @@ function install_from_external_spec() {
         installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
     else
         echo
-        echo "${COLOR_RED}The '${INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}"
+        echo "${COLOR_RED}The '${AIRFLOW_INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}"
         echo
         echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow')${COLOR_RESET}"
         echo

Dockerfile.ci

@@ -936,7 +936,7 @@ function install_from_external_spec() {
         installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
     else
         echo
-        echo "${COLOR_RED}The '${INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}"
+        echo "${COLOR_RED}The '${AIRFLOW_INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}"
         echo
         echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow')${COLOR_RESET}"
         echo

airflow-core/docs/administration-and-deployment/listeners.rst

@@ -24,7 +24,7 @@ You can write listeners to enable Airflow to notify you when events happen.
 .. warning::
 
     Listeners are an advanced feature of Airflow. They are not isolated from the Airflow components they run in, and
-    can slow down or in come cases take down your Airflow instance. As such, extra care should be taken when writing listeners.
+    can slow down or in some cases take down your Airflow instance. As such, extra care should be taken when writing listeners.
 
 Airflow supports notifications for the following events:
 

airflow-core/docs/administration-and-deployment/logging-monitoring/callbacks.rst

@@ -163,3 +163,13 @@ Here's an example of using a custom notifier:
 
 For a list of community-managed Notifiers, see :doc:`apache-airflow-providers:core-extensions/notifications`.
 For more information on writing a custom Notifier, see the :doc:`Notifiers <../../howto/notifications>` how-to page.
+
+Deadline Alert Callbacks
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+In addition to the Dag/task lifecycle callbacks above, Airflow supports **Deadline Alert** callbacks which
+trigger when a Dag run exceeds a configured time threshold. Deadline Alert callbacks use
+:class:`~airflow.sdk.AsyncCallback` (runs in the Triggerer) or :class:`~airflow.sdk.SyncCallback`
+(runs in the executor) and are configured on the Dag via the ``deadline`` parameter.
+
+For full details, see :doc:`/howto/deadline-alerts`.