compatibility-tests/.github/workflows/producer_dbt.yml at 59d2f11e79bcadeeb02280bed2bce74c1c58e52b · OpenLineage/compatibility-tests · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
name: dbt Producer

on:
  workflow_call:
    inputs:
      dbt_release:
        description: "release of dbt-core to use"
        type: string
      ol_release:
        description: "release tag of OpenLineage to use"
        type: string
      get-latest-snapshots:
        description: "Should the artifact be downloaded from maven repo or circleci"
        type: string
  workflow_dispatch:
    inputs:
      dbt_release:
        description: "release of dbt-core to use"
        type: string
        default: "1.8.0"
      ol_release:
        description: "release tag of OpenLineage to use"
        type: string
        default: "1.23.0"
      get-latest-snapshots:
        description: "Should the artifact be downloaded from maven repo or circleci"
        type: string
        default: "false"

jobs:
  run-dbt-tests:
    runs-on: ubuntu-latest

    services:
      postgres:
        image: postgres:15-alpine
        env:
          POSTGRES_USER: testuser
          POSTGRES_PASSWORD: testpass
          POSTGRES_DB: dbt_test
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U testuser -d dbt_test"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Initialize tests
        id: init
        run: |
          scenarios=$(./scripts/get_valid_test_scenarios.sh "producer/dbt/scenarios/" ${{ inputs.dbt_release }} ${{ inputs.ol_release }} )
          if [[ "$scenarios" != "" ]]; then
            echo "scenarios=$scenarios" >> $GITHUB_OUTPUT
            echo "Found scenarios: $scenarios"
          else
            echo "No valid scenarios found for dbt ${{ inputs.dbt_release }} and OL ${{ inputs.ol_release }}"
          fi

      - name: Set up Python 3.12
        if: ${{ steps.init.outputs.scenarios }}
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dbt dependencies
        if: ${{ steps.init.outputs.scenarios }}
        run: |
          python -m pip install --upgrade pip
          pip install dbt-core==${{ inputs.dbt_release }}
          pip install dbt-postgres
          pip install openlineage-dbt==${{ inputs.ol_release }}
          pip install -r producer/dbt/test_runner/requirements.txt

      - name: Set producer output event dir
        if: ${{ steps.init.outputs.scenarios }}
        id: set-producer-output
        run: |
          echo "event_dir=/tmp/dbt-events-$(date +%s%3N)" >> $GITHUB_OUTPUT

      - name: Run dbt scenarios and create OL events
        if: ${{ steps.init.outputs.scenarios }}
        id: run-producer
        continue-on-error: true
        run: |
          set -e
          IFS=';' read -ra scenarios <<< "${{ steps.init.outputs.scenarios }}"

          for scenario in "${scenarios[@]}"
          do
              echo "Running dbt scenario: $scenario"

              if ! python3 producer/dbt/test_runner/cli.py run-scenario \
                --scenario "$scenario" \
                --output-dir "${{ steps.set-producer-output.outputs.event_dir }}"
              then
                echo "Error: dbt scenario failed: $scenario"
                exit 1
              fi

              echo "Finished running scenario: $scenario"
          done

          echo "Finished running all scenarios"

      - name: Validation
        if: ${{ steps.init.outputs.scenarios }}
        uses: ./.github/actions/run_event_validation
        with:
          component: 'dbt'
          producer-dir: 'producer'
          release_tags: ${{ inputs.get-latest-snapshots == 'true' && 'main' || inputs.ol_release }}
          ol_release: ${{ inputs.ol_release }}
          component_release: ${{ inputs.dbt_release }}
          event-directory: ${{ steps.set-producer-output.outputs.event_dir }}
          target-path: 'dbt-${{inputs.dbt_release}}-${{inputs.ol_release}}-report.json'

      - uses: actions/upload-artifact@v4
        if: ${{ steps.init.outputs.scenarios }}
        with:
          name: dbt-${{inputs.dbt_release}}-${{inputs.ol_release}}-report
          path: dbt-${{inputs.dbt_release}}-${{inputs.ol_release}}-report.json
          retention-days: 1