Parameterize source database and add prd workflow

.github/workflows/deploy-dev.yml

@@ -0,0 +1,96 @@
+name: Deploy dabdbt (dev)
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: dev
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
+      DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
+      DATABRICKS_TENANT_ID: ${{ secrets.DATABRICKS_TENANT_ID }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      DATABRICKS_PROFILE: dev-github
+      DBT_DEV_HTTP_PATH: ${{ secrets.DBT_DEV_HTTP_PATH }}
+      AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }}
+      AZURE_STORAGE_CONTAINER: '$web'
+      AZURE_BLOB_PATH: 'dabdbt/index.html'
+      AZURE_BLOB_CONTENT_TYPE: 'text/html; charset=utf-8'
+      AZURE_BLOB_CONTENT_DISPOSITION: inline
+      AZURE_SUBSCRIPTION_ID: '${{ secrets.AZURE_SUBSCRIPTION_ID }}'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Install Databricks CLI
+        run: |
+          curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | bash
+          echo "${HOME}/.databricks/bin" >> "$GITHUB_PATH"
+          echo "${HOME}/bin" >> "$GITHUB_PATH"
+
+      - name: Configure Databricks profile (Service Principal or PAT)
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.databricks
+          if [ -n "${DATABRICKS_CLIENT_ID:-}" ] && [ -n "${DATABRICKS_CLIENT_SECRET:-}" ]; then
+            printf '[%s]\nhost = %s\nclient_id = %s\nclient_secret = %s\n' "${DATABRICKS_PROFILE}" "${DATABRICKS_HOST}" "${DATABRICKS_CLIENT_ID}" "${DATABRICKS_CLIENT_SECRET}" > ~/.databrickscfg
+            if [ -n "${DATABRICKS_TENANT_ID:-}" ]; then
+              printf 'tenant_id = %s\n' "${DATABRICKS_TENANT_ID}" >> ~/.databrickscfg
+            fi
+          elif [ -n "${DATABRICKS_TOKEN:-}" ]; then
+            printf '[%s]\nhost = %s\ntoken = %s\n' "${DATABRICKS_PROFILE}" "${DATABRICKS_HOST}" "${DATABRICKS_TOKEN}" > ~/.databrickscfg
+          else
+            echo "Missing Databricks credentials: set secrets.DATABRICKS_CLIENT_ID/SECRET or secrets.DATABRICKS_TOKEN" >&2
+            exit 1
+          fi
+
+      - name: Deploy bundle to Databricks (dev)
+        working-directory: dabdbt
+        run: |
+          databricks bundle deploy --target dev --profile "${DATABRICKS_PROFILE}"
+
+      - name: Run dabdbt_job (dev)
+        working-directory: dabdbt
+        run: |
+          databricks bundle run dabdbt_job --target dev --profile "${DATABRICKS_PROFILE}"
+
+      - name: Authenticate with Azure
+        uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Set Azure subscription
+        if: env.AZURE_SUBSCRIPTION_ID != ''
+        run: |
+          az account set --subscription "${AZURE_SUBSCRIPTION_ID}"
+
+      - name: Fix static website MIME metadata
+        run: |
+          az storage blob update \
+            --account-name "${AZURE_STORAGE_ACCOUNT}" \
+            --container-name "${AZURE_STORAGE_CONTAINER}" \
+            --name "${AZURE_BLOB_PATH}" \
+            --content-type "${AZURE_BLOB_CONTENT_TYPE}" \
+            --content-disposition "${AZURE_BLOB_CONTENT_DISPOSITION}" \
+            --auth-mode login
+
+

.github/workflows/deploy-prd.yml

@@ -0,0 +1,93 @@
+name: Deploy dabdbt (prd)
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: prd
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
+      DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
+      DATABRICKS_TENANT_ID: ${{ secrets.DATABRICKS_TENANT_ID }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      DATABRICKS_PROFILE: prd-github
+      AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }}
+      AZURE_STORAGE_CONTAINER: '$web'
+      AZURE_BLOB_PATH: 'dabdbt/index.html'
+      AZURE_BLOB_CONTENT_TYPE: 'text/html; charset=utf-8'
+      AZURE_BLOB_CONTENT_DISPOSITION: inline
+      AZURE_SUBSCRIPTION_ID: '${{ secrets.AZURE_SUBSCRIPTION_ID }}'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Install Databricks CLI
+        run: |
+          curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | bash
+          echo "${HOME}/.databricks/bin" >> "$GITHUB_PATH"
+          echo "${HOME}/bin" >> "$GITHUB_PATH"
+
+      - name: Configure Databricks profile (Service Principal or PAT)
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.databricks
+          if [ -n "${DATABRICKS_CLIENT_ID:-}" ] && [ -n "${DATABRICKS_CLIENT_SECRET:-}" ]; then
+            printf '[%s]\nhost = %s\nclient_id = %s\nclient_secret = %s\n' "${DATABRICKS_PROFILE}" "${DATABRICKS_HOST}" "${DATABRICKS_CLIENT_ID}" "${DATABRICKS_CLIENT_SECRET}" > ~/.databrickscfg
+            if [ -n "${DATABRICKS_TENANT_ID:-}" ]; then
+              printf 'tenant_id = %s\n' "${DATABRICKS_TENANT_ID}" >> ~/.databrickscfg
+            fi
+          elif [ -n "${DATABRICKS_TOKEN:-}" ]; then
+            printf '[%s]\nhost = %s\ntoken = %s\n' "${DATABRICKS_PROFILE}" "${DATABRICKS_HOST}" "${DATABRICKS_TOKEN}" > ~/.databrickscfg
+          else
+            echo "Missing Databricks credentials: set secrets.DATABRICKS_CLIENT_ID/SECRET or secrets.DATABRICKS_TOKEN" >&2
+            exit 1
+          fi
+
+      - name: Deploy bundle to Databricks (prd)
+        working-directory: dabdbt
+        run: |
+          databricks bundle deploy --target prd --profile "${DATABRICKS_PROFILE}"
+
+      - name: Run dabdbt_job (prd)
+        working-directory: dabdbt
+        run: |
+          databricks bundle run dabdbt_job --target prd --profile "${DATABRICKS_PROFILE}"
+
+      - name: Authenticate with Azure
+        uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Set Azure subscription
+        if: env.AZURE_SUBSCRIPTION_ID != ''
+        run: |
+          az account set --subscription "${AZURE_SUBSCRIPTION_ID}"
+
+      - name: Fix static website MIME metadata
+        run: |
+          az storage blob update \
+            --account-name "${AZURE_STORAGE_ACCOUNT}" \
+            --container-name "${AZURE_STORAGE_CONTAINER}" \
+            --name "${AZURE_BLOB_PATH}" \
+            --content-type "${AZURE_BLOB_CONTENT_TYPE}" \
+            --content-disposition "${AZURE_BLOB_CONTENT_DISPOSITION}" \
+            --auth-mode login

dabdbt/databricks.yml

@@ -13,6 +13,9 @@ variables:
   prod_root_path:
     description: Workspace path for production bundle deployment
     default: /Workspace/Users/anselmoborges@gmail.com/.bundle/${bundle.name}/${bundle.target}
+  source_olist_env:
+    description: Target suffix used by Unity Catalog foreign catalog
+    default: ${bundle.target}
   prod_owner_user:
     description: User principal that will retain CAN_MANAGE in production
     default: anselmoborges@gmail.com
@@ -28,7 +31,7 @@ targets:
     mode: development
     default: true
     workspace:
-      host: https://adb-1293581597272291.11.azuredatabricks.net
+      host: https://adb-1293581597272291.11.azuredatabricks.net/
 
   prd:
     mode: production