data-engineering/.github/workflows/pluto_build.yml at main · NYCPlanning/data-engineering · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
name: PLUTO - 🏗️ Build

on:
  workflow_call:
    inputs:
      image_tag:
        type: string
        required: false
      build_name:
        type: string
        required: true
      recipe_file:
        type: string
        required: true
      version:
        type: string
        required: false
      plan_command:
        type: string
        default: recipe
      dev_bucket:
        type: string
        required: false
      test_severity:
        type: string
        default: 'error'

jobs:
  build:
    name: Build PLUTO
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: products/pluto/pluto_build
    container:
      image: nycplanning/build-base:${{ inputs.image_tag || 'latest' }}
    env:
      BUILD_ENGINE_DB: db-pluto
      BUILD_NAME: ${{ inputs.build_name }}
      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
      TEST_SEVERITY: ${{ inputs.test_severity }}
    steps:
    - uses: actions/checkout@v4

    - name: Load Secrets
      uses: 1password/load-secrets-action@v1
      with:
        export-env: true
      env:
        OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
        AWS_S3_ENDPOINT: "op://Data Engineering/DO_keys/AWS_S3_ENDPOINT"
        AWS_SECRET_ACCESS_KEY: "op://Data Engineering/DO_keys/AWS_SECRET_ACCESS_KEY"
        AWS_ACCESS_KEY_ID: "op://Data Engineering/DO_keys/AWS_ACCESS_KEY_ID"
        BUILD_ENGINE_SERVER: "op://Data Engineering/EDM_DATA/server_url"
        BUILD_ENGINE_HOST: "op://Data Engineering/EDM_DATA/server"
        BUILD_ENGINE_USER: "op://Data Engineering/EDM_DATA/username"
        BUILD_ENGINE_PASSWORD: "op://Data Engineering/EDM_DATA/password"
        BUILD_ENGINE_PORT: "op://Data Engineering/EDM_DATA/port"
        EDM_DATA: "op://Data Engineering/EDM_DATA/defaultdb_url"

    - name: Finish container setup
      working-directory: ./
      run: ./bash/docker_container_setup.sh

    - name: Set build environment variables
      working-directory: ./
      run: ./bash/build_env_setup.sh

    - name: Plan build
      working-directory: products/pluto
      run: python3 -m dcpy.lifecycle.builds.plan ${{ inputs.plan_command }}

    - name: Set recipe env vars
      working-directory: ./
      run: source ./bash/export_recipe_env.sh products/pluto/${{ inputs.recipe_file
        }}.lock.yml

    - name: Dataloading
      working-directory: products/pluto
      run: python3 -m dcpy lifecycle builds load load --recipe-path ${{ inputs.recipe_file
        }}.lock.yml --cache-schema recipe_cache --cached-entity-type view

    - name: Load Local Data and DBT Staging
      run: ./01a_dbt_staging.sh

    - name: building ...
      run: ./02_build.sh

    - name: apply corrections
      run: ./03_corrections.sh

    - name: Archive
      run: ./04_archive.sh

    - name: QAQC
      run: ./05_qaqc.sh

    - name: Export and Upload
      run: ./06_export.sh

    - name: Custom QAQC
      working-directory: products/pluto
      run: ./pluto_build/07_custom_qaqc.sh