feature: adjust the cost import to support a date range by adding a s… #94
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy to Cloud Run Job | |
| env: | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| CICD_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} | |
| DBT_SERVICE_ACCOUNT: ${{ secrets.DBT_SERVICE_ACCOUNT_EMAIL }} | |
| MANIFEST_BUCKET: ${{ secrets.GCP_GCS_BUCKET }} | |
| TARGET: prod | |
| PROJECT_ID: mezo-portal-data | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| types: | |
| - closed | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write | |
| contents: write | |
| jobs: | |
| dbt-build: | |
| runs-on: ubuntu-latest | |
| if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' | |
| env: | |
| DBT_PROJECT_DIR: . | |
| steps: | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| - name: Set up Python | |
| run: uv python install | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install the project | |
| run: uv sync --dev | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.CICD_SERVICE_ACCOUNT }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Copy manifest.json from Google Cloud Storage | |
| id: check-manifest | |
| continue-on-error: true | |
| run: | | |
| mkdir -p target-base | |
| gcloud storage cp gs://${{ secrets.GCP_GCS_BUCKET }}/manifest/manifest.json target-base/ | |
| - name: Run dbt debug | |
| run: | | |
| uv run dbt debug --target ${{ env.TARGET }} | |
| - name: Run dbt deps | |
| run: | | |
| uv run dbt deps --target ${{ env.TARGET }} | |
| - name: Set full refresh flag | |
| id: set-full-refresh-flag | |
| run: | | |
| if echo "${{ github.event.pull_request.body }}" | grep -q "#full-refresh"; then | |
| echo "DBT_FULL_REFRESH=--full-refresh" >> $GITHUB_OUTPUT | |
| else | |
| echo "DBT_FULL_REFRESH=" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Run dbt build | |
| run: | | |
| uv run dbt run-operation stage_external_sources | |
| uv run dbt build --select state:modified+ --target ${{ env.TARGET }} --defer --state target-base --indirect-selection cautious ${{ steps.set-full-refresh-flag.outputs.DBT_FULL_REFRESH }} | |
| - name: Generate dbt documentation | |
| run: | | |
| uv run dbt docs generate | |
| - name: Move dbt docs to a temporary location | |
| run: | | |
| mv target /tmp/ | |
| - name: Checkout gh-pages branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: 'gh-pages' | |
| - name: Copy dbt docs into repo | |
| run: | | |
| cp -r /tmp/target/* ./ | |
| - name: Commit and push updated documentation | |
| uses: stefanzweifel/git-auto-commit-action@v6 | |
| with: | |
| branch: gh-pages | |
| commit_message: Update documentation | |
| file_pattern: 'index.html manifest.json catalog.json' | |
| push_options: '--force' | |
| - name: Upload manifest to GCS | |
| if: always() | |
| run: | | |
| if [ -f target/manifest.json ]; then | |
| gcloud storage cp target/manifest.json gs://${{ env.MANIFEST_BUCKET}}/manifest/manifest.json | |
| else | |
| echo "manifest.json not found, skipping upload." | |
| fi | |
| build-container-and-deploy: | |
| needs: dbt-build | |
| runs-on: ubuntu-latest | |
| env: | |
| SERVICE_NAME: mezo-prod-dp-dwh-dbt-crj-0 | |
| REGION: europe-west3 | |
| steps: | |
| - name: Set env var for image | |
| run: echo "IMAGE=europe-west3-docker.pkg.dev/${{ env.PROJECT_ID }}/dbt-repo/dbt-mezo:${{ github.sha }}" >> $GITHUB_ENV | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Authenticate to Google Cloud (WIF) | |
| id: 'auth' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| workload_identity_provider: '${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}' | |
| service_account: '${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}' | |
| token_format: 'access_token' | |
| create_credentials_file: true | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| project_id: ${{ env.PROJECT_ID }} | |
| - name: Configure Docker for Artifact Registry | |
| run: | | |
| gcloud auth configure-docker europe-west3-docker.pkg.dev | |
| - name: Build and push Docker image | |
| run: | | |
| docker build --platform linux/amd64 -t $IMAGE . | |
| docker push $IMAGE | |
| - name: Deploy Cloud Run Job | |
| run: | | |
| gcloud run jobs deploy $SERVICE_NAME \ | |
| --image $IMAGE \ | |
| --tasks 1 \ | |
| --set-env-vars SLEEP_MS=10000,FAIL_RATE=0.1,DBT_SERVICE_ACCOUNT=${{ secrets.DBT_SERVICE_ACCOUNT_EMAIL }},GCS_BUCKET=${{ secrets.GCP_GCS_BUCKET }} \ | |
| --max-retries 5 \ | |
| --region $REGION \ | |
| --project $PROJECT_ID \ | |
| --memory 1Gi \ | |
| --service-account ${{ secrets.CLOUDRUN_SERVICE_ACCOUNT_EMAIL }} |