Merge origin/master into feat/omni-source

- Resolve setup.py extras conflict: keep omni extra; glue includes sqlglot_lib (master)
- Add [project.optional-dependencies] omni group to pyproject.toml (pip install .[omni])

Made-with: Cursor

Author: bearsandrhinos

.gitattributes

@@ -3,3 +3,4 @@
 *.tsx text eol=lf
 gradlew text eol=lf
 metadata-utils/src/test/resources/filterQuery/* text eol=lf
+

.github/actions/ci-optimization/action.yml

@@ -11,6 +11,9 @@ outputs:
   backend-only:
     description: "Backend only change"
     value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
+  connector-source-only:
+    description: "Only connector source/test files changed (no ingestion core, no backend, no frontend)"
+    value: ${{ steps.filter.outputs.ingestion == 'true' && steps.filter.outputs.ingestion-core == 'false' && steps.filter.outputs.backend == 'false' && steps.filter.outputs.frontend == 'false' }}
   backend-change:
     description: "Backend code has changed"
     value: ${{ steps.filter.outputs.backend == 'true' ||  steps.trigger.outputs.trigger == 'manual' }}
@@ -51,7 +54,7 @@ runs:
         else
           echo "trigger=pr" >> $GITHUB_OUTPUT
         fi
-    - uses: dorny/paths-filter@v3
+    - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
       id: filter
       with:
         token: "" # Empty token forces it to use raw git commands.
@@ -66,6 +69,27 @@ runs:
             - "metadata-models/**"
             - "docker/datahub-ingestion-base/**"
             - "docker/datahub-ingestion/**"
+          ingestion-core:
+            - "metadata-ingestion/setup.py"
+            - "metadata-ingestion/pyproject.toml"
+            - "metadata-ingestion/constraints.txt"
+            - "metadata-ingestion/src/datahub/api/**"
+            - "metadata-ingestion/src/datahub/cli/**"
+            - "metadata-ingestion/src/datahub/configuration/**"
+            - "metadata-ingestion/src/datahub/emitter/**"
+            - "metadata-ingestion/src/datahub/ingestion/api/**"
+            - "metadata-ingestion/src/datahub/ingestion/graph/**"
+            - "metadata-ingestion/src/datahub/ingestion/run/**"
+            - "metadata-ingestion/src/datahub/ingestion/sink/**"
+            - "metadata-ingestion/src/datahub/metadata/**"
+            - "metadata-ingestion/src/datahub/sdk/**"
+            - "metadata-ingestion/src/datahub/sql_parsing/**"
+            - "metadata-ingestion/src/datahub/telemetry/**"
+            - "metadata-ingestion/src/datahub/utilities/**"
+            - "metadata-ingestion-modules/**"
+            - "metadata-models/**"
+            - "docker/datahub-ingestion-base/**"
+            - "docker/datahub-ingestion/**"
           ingestion-base:
             - "docker/datahub-ingestion-base/**"
           docker:

.github/actions/determine-runners/action.yml

@@ -0,0 +1,116 @@
+name: Determine Runners
+description: "Centralized steps to determine which runners are available"
+# Refer: https://depot.dev/docs/github-actions/runner-types and https://docs.github.com/en/actions/reference/runners/github-hosted-runners for details of various available runners.
+
+# Prefer pinning the runner to specific OS version to avoid surprise breakages due to new OS release
+
+# 1. Running on a PR from a fork (community contribution). We use github runners, unless the "depot" label exists -- in which case, we run it on depot.author:
+#Note: Concurrency is lower when using github runners, queue times can be longer, test time is longer due to fewer parallel jobs.
+# 2. Running on a PR from a branch in the datahub-project org and push/schedule events on master.
+#    Depot is used here for remote container builds in base_build and also for all runners. Depot runners support unlimited concurrency
+#    and hence short queue times and higher parallelism of smoke tests
+
+inputs:
+  is-community-pr:
+    description: "Whether this is a PR from a fork repository"
+    required: false
+    default: "true"
+  has-depot-label:
+    description: "Whether the PR has the 'depot' label (allows Community PRs to opt-in to Depot runners)"
+    required: false
+    default: "false"
+
+outputs:
+  depot-config-set:
+    description: "true if required depot vars are set"
+    value: ${{ steps.depot-config-set.outputs.value }}
+  default-runner:
+    description: "Default runner commonly used for regular jobs. Appropriately set if Depot runners are available"
+    value: ${{ steps.default-runner.outputs.value }}
+  default-gh-runner:
+    description: "The runner to use for most regular jobs to be run on  Github Hosted Runners"
+    value: ubuntu-24.04
+  default-depot-runner:
+    description: "The runner to use for most regular jobs to be run on Depot Runners. Use when you always want to run a job on Depot runners"
+    value: "depot-ubuntu-24.04"
+  small-runner:
+    description: "Appropriately set small runner based on Depot availability"
+    value: ${{ steps.small-runner.outputs.value }}
+  depot-runner-medium:
+    description: "depot-ubuntu-24.04-2 Runner. Use for medium jobs"
+    value: "depot-ubuntu-24.04-2"
+  depot-runner-large:
+    description: "depot-ubuntu-24.04-4 Runner. Use for heavy jobs"
+    value: "depot-ubuntu-24.04-4"
+  depot-label-runner:
+    description: "Set to depot-runner if depot label is added to Community PR"
+    value: ${{ steps.depot-label-small-runner.outputs.value }}
+  depot-label-small-runner:
+    description: "Set to depot-runner if depot label is added to Community PR"
+    value: ${{ steps.depot-label-small-runner.outputs.value }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check Depot Configuration is set
+      id: depot-config-set
+      shell: bash
+      run: |
+        if [[ -n "$DEPOT_PROJECT_ID" ]]; then
+          echo "value=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "value=false" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Set default-runner
+      id: default-runner
+      shell: bash
+      env:
+        IS_COMMUNITY_PR: ${{ inputs.is-community-pr }}
+        DEPOT_CONFIG_SET: ${{ steps.depot-config-set.outputs.value }}
+      # Use depot for internal branches and push/schedule events; fall back to GitHub-hosted for fork PRs.
+      run: |
+        if [[ "$IS_COMMUNITY_PR" == "false" && "$DEPOT_CONFIG_SET" == "true" ]]; then
+          echo "value=depot-ubuntu-24.04" >> "$GITHUB_OUTPUT"
+        else
+          echo "value=ubuntu-24.04" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Set small-runner
+      id: small-runner
+      shell: bash
+      env:
+        IS_COMMUNITY_PR: ${{ inputs.is-community-pr }}
+        DEPOT_CONFIG_SET: ${{ steps.depot-config-set.outputs.value }}
+      # depot-small for internal, ubuntu-latest for forks.
+      run: |
+        if [[ "$IS_COMMUNITY_PR" == "false" && "$DEPOT_CONFIG_SET" == "true" ]]; then
+          echo "value=depot-ubuntu-24.04-small" >> "$GITHUB_OUTPUT"
+        else
+          echo "value=ubuntu-24.04" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Set depot-label-small-runner
+      id: depot-label-small-runner
+      env:
+        HAS_DEPOT_LABEL: ${{ inputs.has-depot-label }}
+      shell: bash
+      # Community (fork) PRs can opt-in to Depot runners by adding the 'depot' label.
+      run: |
+        if [[ "$HAS_DEPOT_LABEL" == "true" ]]; then
+          echo "value=depot-ubuntu-24.04-small" >> "$GITHUB_OUTPUT"
+        else
+          echo "value=ubuntu-24.04" >> "$GITHUB_OUTPUT"
+        fi
+    - name: Set depot-label-runner
+      id: depot-label-runner
+      shell: bash
+      env:
+        HAS_DEPOT_LABEL: ${{ inputs.has-depot-label }}
+      # Community (fork) PRs can opt-in to Depot runners by adding the 'depot' label.
+      run: |
+        if [[ "$HAS_DEPOT_LABEL" == "true" ]]; then
+          echo "value=depot-ubuntu-24.04" >> "$GITHUB_OUTPUT"
+        else
+          echo "value=ubuntu-24.04" >> "$GITHUB_OUTPUT"
+        fi

.github/actions/docker-custom-build-and-push/action.yml

@@ -53,7 +53,7 @@ runs:
   steps:
     - name: Docker meta
       id: docker_meta
-      uses: docker/metadata-action@v5
+      uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
       with:
         images: ${{ inputs.images }}
         flavor: |
@@ -78,7 +78,7 @@ runs:
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -97,32 +97,32 @@ runs:
         cache-to: |
           type=inline
     - name: Upload image locally for testing (if not publishing)
-      uses: ishworkh/container-image-artifact-upload@v2.0.0
+      uses: ishworkh/container-image-artifact-upload@5d71a2417f0576fa11fe770fb04ece58c4587714 # v2.0.0
       if: ${{ inputs.publish != 'true' }}
       with:
         image: ${{ steps.single_tag.outputs.SINGLE_TAG }}
         retention_days: "2"
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
       if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
       if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
     - name: Setup Depot CLI
-      uses: depot/setup-action@v1
+      uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1
       if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
     - name: Login to DockerHub
-      uses: docker/login-action@v3
+      uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
 
     # Depot variant.
     - name: Build and Push Multi-Platform image
-      uses: depot/build-push-action@v1
+      uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1
       if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
       with:
         project: ${{ inputs.depot-project }}
@@ -140,7 +140,7 @@ runs:
           type=inline
 
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
       if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
       with:
         context: ${{ inputs.context }}

.github/actions/report-test-results/action.yml

@@ -18,15 +18,15 @@ runs:
   steps:
     - name: Upload test results
       if: (!cancelled())
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: ${{ inputs.artifact-name }}
         path: ${{ inputs.test-results-paths }}
         retention-days: 7
 
     - name: Publish test results
       if: (!cancelled())
-      uses: test-summary/action@v2
+      uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2
       with:
         paths: ${{ inputs.junit-file-globs }}
         show: fail

.github/path-filters/lint-jobs-filters.yml

@@ -0,0 +1,31 @@
+lint-job:
+  - modified: ".github/workflows/lint-jobs.yml"
+  - ".github/path-filters/lint-job-filter.yml"
+markdown:
+  - added|modified: "**/*.md"
+github-actions:
+  - added|modified: ".github/**/*.yml"
+  - added|modified: ".github/**/*.yaml"
+workflow-files:
+  - added|modified: ".github/workflows/**"
+workflow-validation:
+  - ".github/workflows/**"
+  - ".github/scripts/validate_post_workflow_list.py"
+code-check-sources:
+  - "metadata-io/**"
+  - "datahub-web-react/**"
+  - "metadata-service/war/src/main/resources/boot/policies.json"
+  - "**/build.gradle"
+  - "**/build.gradle.kts"
+  - "**/gradle.lockfile"
+  - ".github/scripts/check_*.py"
+smoke-test-python:
+  - added|modified: "smoke-test/**/*.py"
+  - "smoke-test/pyproject.toml"
+  - "smoke-test/requirements.txt"
+  - "smoke-test/build.gradle"
+smoke-test-cypress:
+  - "smoke-test/tests/cypress/**"
+datahub-web-react:
+  - added|modified: "datahub-web-react/**"
+  - "!datahub-web-react/**/*.md"

.github/pull_request_template.md

@@ -11,7 +11,7 @@ Before you submit your PR, please go through the checklist below:
 - [ ] For any breaking change/potential downtime/deprecation/big changes an entry has been made in [Updating DataHub](https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md)
 
 
-Allowed Types in PR Title: _feat_, _fix_, _refactor_, _docs_, _test_, _perf_, _style_, _build_, _ci_
+Allowed Types in PR Title: _feat_, _fix_, _refactor_, _docs_, _test_, _perf_, _style_, _build_, _ci_, _chore_
 
 
 -->