|
| 1 | +name: dbt Producer |
| 2 | + |
| 3 | +on: |
| 4 | + workflow_call: |
| 5 | + inputs: |
| 6 | + dbt_release: |
| 7 | + description: "release of dbt-core to use" |
| 8 | + type: string |
| 9 | + ol_release: |
| 10 | + description: "release tag of OpenLineage to use" |
| 11 | + type: string |
| 12 | + get-latest-snapshots: |
| 13 | + description: "Should the artifact be downloaded from maven repo or circleci" |
| 14 | + type: string |
| 15 | + workflow_dispatch: |
| 16 | + inputs: |
| 17 | + dbt_release: |
| 18 | + description: "release of dbt-core to use" |
| 19 | + type: string |
| 20 | + default: "1.8.0" |
| 21 | + ol_release: |
| 22 | + description: "release tag of OpenLineage to use" |
| 23 | + type: string |
| 24 | + default: "1.23.0" |
| 25 | + get-latest-snapshots: |
| 26 | + description: "Should the artifact be downloaded from maven repo or circleci" |
| 27 | + type: string |
| 28 | + default: "false" |
| 29 | + |
| 30 | +jobs: |
| 31 | + run-dbt-tests: |
| 32 | + runs-on: ubuntu-latest |
| 33 | + |
| 34 | + services: |
| 35 | + postgres: |
| 36 | + image: postgres:15-alpine |
| 37 | + env: |
| 38 | + POSTGRES_USER: testuser |
| 39 | + POSTGRES_PASSWORD: testpass |
| 40 | + POSTGRES_DB: dbt_test |
| 41 | + ports: |
| 42 | + - 5432:5432 |
| 43 | + options: >- |
| 44 | + --health-cmd "pg_isready -U testuser -d dbt_test" |
| 45 | + --health-interval 10s |
| 46 | + --health-timeout 5s |
| 47 | + --health-retries 5 |
| 48 | + |
| 49 | + steps: |
| 50 | + - name: Checkout code |
| 51 | + uses: actions/checkout@v4 |
| 52 | + |
| 53 | + - name: Initialize tests |
| 54 | + id: init |
| 55 | + run: | |
| 56 | + scenarios=$(./scripts/get_valid_test_scenarios.sh "producer/dbt/scenarios/" ${{ inputs.dbt_release }} ${{ inputs.ol_release }} ) |
| 57 | + if [[ "$scenarios" != "" ]]; then |
| 58 | + echo "scenarios=$scenarios" >> $GITHUB_OUTPUT |
| 59 | + echo "Found scenarios: $scenarios" |
| 60 | + else |
| 61 | + echo "No valid scenarios found for dbt ${{ inputs.dbt_release }} and OL ${{ inputs.ol_release }}" |
| 62 | + fi |
| 63 | +
|
| 64 | + - name: Set up Python 3.12 |
| 65 | + if: ${{ steps.init.outputs.scenarios }} |
| 66 | + uses: actions/setup-python@v5 |
| 67 | + with: |
| 68 | + python-version: "3.12" |
| 69 | + |
| 70 | + - name: Install dbt dependencies |
| 71 | + if: ${{ steps.init.outputs.scenarios }} |
| 72 | + run: | |
| 73 | + python -m pip install --upgrade pip |
| 74 | + pip install dbt-core==${{ inputs.dbt_release }} |
| 75 | + pip install dbt-postgres |
| 76 | + pip install openlineage-dbt==${{ inputs.ol_release }} |
| 77 | +
|
| 78 | + - name: Set producer output event dir |
| 79 | + if: ${{ steps.init.outputs.scenarios }} |
| 80 | + id: set-producer-output |
| 81 | + run: | |
| 82 | + echo "event_dir=/tmp/dbt-events-$(date +%s%3N)" >> $GITHUB_OUTPUT |
| 83 | +
|
| 84 | + - name: Run dbt scenarios and create OL events |
| 85 | + if: ${{ steps.init.outputs.scenarios }} |
| 86 | + id: run-producer |
| 87 | + continue-on-error: true |
| 88 | + run: | |
| 89 | + set -e |
| 90 | + IFS=';' read -ra scenarios <<< "${{ steps.init.outputs.scenarios }}" |
| 91 | + |
| 92 | + for scenario in "${scenarios[@]}" |
| 93 | + do |
| 94 | + echo "Running dbt scenario: $scenario" |
| 95 | + |
| 96 | + mkdir -p "${{ steps.set-producer-output.outputs.event_dir }}/$scenario" |
| 97 | + bash producer/dbt/scenarios/$scenario/test/run.sh "${{ steps.set-producer-output.outputs.event_dir }}/$scenario" |
| 98 | +
|
| 99 | + echo "Finished running scenario: $scenario" |
| 100 | + done |
| 101 | +
|
| 102 | + echo "Finished running all scenarios" |
| 103 | +
|
| 104 | + - uses: actions/upload-artifact@v4 |
| 105 | + if: ${{ steps.init.outputs.scenarios }} |
| 106 | + with: |
| 107 | + name: dbt-${{inputs.dbt_release}}-${{inputs.ol_release}}-events |
| 108 | + path: ${{ steps.set-producer-output.outputs.event_dir }} |
| 109 | + retention-days: 1 |
| 110 | + |
| 111 | + - name: Validation |
| 112 | + if: ${{ steps.init.outputs.scenarios }} |
| 113 | + uses: ./.github/actions/run_event_validation |
| 114 | + with: |
| 115 | + component: 'dbt' |
| 116 | + producer-dir: 'producer' |
| 117 | + release_tags: ${{ inputs.get-latest-snapshots == 'true' && 'main' || inputs.ol_release }} |
| 118 | + ol_release: ${{ inputs.ol_release }} |
| 119 | + component_release: ${{ inputs.dbt_release }} |
| 120 | + event-directory: ${{ steps.set-producer-output.outputs.event_dir }} |
| 121 | + target-path: 'dbt-${{inputs.dbt_release}}-${{inputs.ol_release}}-report.json' |
| 122 | + |
| 123 | + - uses: actions/upload-artifact@v4 |
| 124 | + if: ${{ steps.init.outputs.scenarios }} |
| 125 | + with: |
| 126 | + name: dbt-${{inputs.dbt_release}}-${{inputs.ol_release}}-report |
| 127 | + path: dbt-${{inputs.dbt_release}}-${{inputs.ol_release}}-report.json |
| 128 | + retention-days: 1 |
0 commit comments