main_new_release.yml

name: Release check

on:
  schedule:
    - cron: "0 3 * * *"
  workflow_dispatch:
    inputs:
      run_dataplex:
        description: 'Run Dataplex tests'
        required: false
        default: 'true'
      run_spark_dataproc:
        description: 'Run Spark Dataproc tests'
        required: false
        default: 'true'
      run_hive_dataproc:
        description: 'Run Hive Dataproc tests'
        required: false
        default: 'true'
      run_dbt:
        description: 'Run DBT tests'
        required: false
        default: 'true'
      openlineage_release:
        description: 'Override OpenLineage release version'
        required: false
      spark_matrix:
        description: 'Overwrite matrix for spark tests'
        required: false
      hive_matrix:
        description: 'Overwrite matrix for hive tests'
        required: false
      dbt_matrix:
        description: 'Overwrite matrix for hive tests'
        required: false

permissions:
  id-token: write
  contents: write
  pull-requests: write
  issues: write

jobs:
  initialize_workflow:
    runs-on: ubuntu-latest
    outputs:
      run_dataplex: ${{ github.event.inputs.run_dataplex || 'true' }}
      run_spark_dataproc: ${{ github.event.inputs.run_spark_dataproc || 'true' }}
      run_hive_dataproc: ${{ github.event.inputs.run_hive_dataproc || 'true' }}
      run_dbt: ${{ github.event.inputs.run_dbt || 'true' }}
      openlineage_release: ${{ github.event.inputs.openlineage_release || steps.select-components.outputs.ol_release }}
      spark_matrix: ${{ github.event.inputs.spark_matrix || steps.set-matrix-values.outputs.spark_dataproc_matrix }}
      hive_matrix: ${{ github.event.inputs.hive_matrix || steps.set-matrix-values.outputs.hive_dataproc_matrix }}
      dbt_matrix: ${{ github.event.inputs.dbt_matrix || steps.set-matrix-values.outputs.dbt_matrix }}
      execution_time: ${{ steps.get-execution-time.outputs.execution_time }}
    steps:
      - name: Get execution time
        id: get-execution-time
        run: echo "execution_time=$(date +'%Y%d%m%H%M')" >> $GITHUB_OUTPUT

      - name: Checkout code
        uses: actions/checkout@v4

        # there should be part about checking not only the OL version but also the producers and consumers
        # there is no way to check for new releases of spark_dataproc or dataplex so we just run their tests everytime
        # also normally new release of OL should trigger all producer tests but for now they are run anyway so no need to trigger
      - name: Select components to run
        id: select-components
        run: ./scripts/select_components.sh

      - uses: actions/upload-artifact@v4
        if: steps.select-components.outputs.releases_updated == 'true'
        with:
          name: updated-releases
          path: generated-files/updated-releases.json
          retention-days: 1

      - name: set-matrix-values
        id: set-matrix-values
        run: |
          get_matrix() {
            local component="$1"
            local releases_file="./generated-files/updated-releases.json"
            local versions_file="producer/$component/versions.json"

            [[ -f "$releases_file" && -f "$versions_file" ]] || {
              echo "Missing releases.json or versions.json" >&2
              return 1
            }

            local latest_ol=$(jq -r '.[] | select(.name == "openlineage") | .latest_version' "$releases_file")
            local latest_comp=$(jq -r --arg name "$component" '.[] | select(.name == $name) | .latest_version' "$releases_file")

            jq -c --arg ol "$latest_ol" --arg comp "$latest_comp" '
              .openlineage_versions |= (if $ol != "" and index($ol) == null then . + [$ol] else . end) |
              .component_version |= (if $comp != "" and index($comp) == null then . + [$comp] else . end)
            ' "$versions_file"
          }

          echo "spark_dataproc_matrix=$(get_matrix spark_dataproc)" >> $GITHUB_OUTPUT
          echo "hive_dataproc_matrix=$(get_matrix hive_dataproc)" >> $GITHUB_OUTPUT
          echo "dbt_matrix=$(get_matrix dbt)" >> $GITHUB_OUTPUT

  ######## COMPONENT VALIDATION ########

  dataplex:
    needs: initialize_workflow
    if: ${{ needs.initialize_workflow.outputs.run_dataplex == 'true' }}
    uses: ./.github/workflows/consumer_dataplex.yml
    secrets:
      gcpKey: ${{ secrets.GCP_SA_KEY }}
    with:
      release: ${{ needs.initialize_workflow.outputs.openlineage_release }}

  spark-dataproc:
    needs: initialize_workflow
    if: ${{ needs.initialize_workflow.outputs.run_spark_dataproc == 'true' }}
    uses: ./.github/workflows/producer_spark_dataproc.yml
    strategy:
      matrix: ${{ fromJson(needs.initialize_workflow.outputs.spark_matrix) }}
    secrets:
      gcpKey: ${{ secrets.GCP_SA_KEY }}
      postgresqlUser: ${{ secrets.POSTGRESQL_USER }}
      postgresqlPassword: ${{ secrets.POSTGRESQL_PASSWORD }}
    with:
      ol_release: ${{ matrix.openlineage_versions }}
      spark_release: ${{ matrix.component_version }}
      get-latest-snapshots: 'false'

  hive-dataproc:
    needs: initialize_workflow
    if: ${{ needs.initialize_workflow.outputs.run_hive_dataproc == 'true' }}
    uses: ./.github/workflows/producer_hive_dataproc.yml
    strategy:
      matrix: ${{ fromJson(needs.initialize_workflow.outputs.hive_matrix) }}
    secrets:
      gcpKey: ${{ secrets.GCP_SA_KEY }}
    with:
      ol_release: ${{ matrix.openlineage_versions }}
      component_release: ${{ matrix.component_version }}
      get-latest-snapshots: 'false'

  dbt:
    needs: initialize_workflow
    if: ${{ needs.initialize_workflow.outputs.run_dbt == 'true' }}
    uses: ./.github/workflows/producer_dbt.yml
    strategy:
      matrix: ${{ fromJson(needs.initialize_workflow.outputs.dbt_matrix) }}
    with:
      ol_release: ${{ matrix.openlineage_versions }}
      dbt_release: ${{ matrix.component_version }}
      get-latest-snapshots: 'false'

  ######## COLLECTION OF REPORTS AND EXECUTE APPROPRIATE ACTIONS ########

  collect-and-compare-reports:
    needs:
      - initialize_workflow
      - dataplex
      - spark-dataproc
      - hive-dataproc
      - dbt
    if: ${{ !failure() }}
    uses: ./.github/workflows/collect_and_compare_reports.yml

  notify-maintainers:
    needs:
      - initialize_workflow
      - collect-and-compare-reports
    if: ${{ !failure() }}
    uses: ./.github/workflows/notify_maintainers.yml
    with:
      workflow-type: 'release'
      trigger-type: ${{ github.event.inputs.openlineage_release && 'manual' || 'scheduled' }}
      execution-time: ${{ needs.initialize_workflow.outputs.execution_time }}

  update-repo-files:
    needs:
      - initialize_workflow
      - notify-maintainers
    if: ${{ !failure() }}
    uses: ./.github/workflows/update_repo_files.yml
    secrets:
      CI_PUSH_TOKEN: ${{ secrets.CI_PUSH_TOKEN }}
    with:
     workflow-type: 'release'
     trigger-type: ${{ github.event.inputs.openlineage_release && 'manual' || 'scheduled' }}
     execution-time: ${{ needs.initialize_workflow.outputs.execution_time }}

  generate-compatibility-tables:
    needs: update-repo-files
    if: ${{ !failure() && needs.update-repo-files.outputs.changes_made == 'true' }}
    uses: ./.github/workflows/generate_compatibility_tables.yml

  update-site:
    needs:
      - update-repo-files
      - generate-compatibility-tables
    if: ${{ !failure() && needs.update-repo-files.outputs.changes_made == 'true' }}
    uses: ./.github/workflows/update_site.yml
    secrets:
      CI_PUSH_TOKEN: ${{ secrets.OL_SITE_PUSH_TOKEN }}