Create inference-integration-deployment.yml

Author: kaylawilding

.github/workflows/inference-integration-deployment.yml

@@ -0,0 +1,115 @@
+name: inference-integration-deployment
+
+on:
+# dev or main here to work out
+  pull-request:
+    branches:
+      - develop
+
+jobs:
+  integration-test-dev:
+    name: Run Integration Tests on Databricks Dev Workspace
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          uv venv
+          uv pip install .
+
+      - name: Set up Databricks CLI for Dev Workspace
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_DEV_TOKEN }}
+        run: |
+          databricks auth login --host "$DATABRICKS_HOST" --token "$DATABRICKS_TOKEN"
+          databricks auth validate
+
+      - name: Run Integration Test Job on Dev
+        env:
+          PROFILE: ${{ secrets.DEV_PROFILE_NAME }}
+          DB_WORKSPACE: ${{ secrets.DEV_DB_WORKSPACE }}
+          SA_EXECUTER: ${{ secrets.DEV_SERVICE_ACCOUNT_EXECUTER }}
+          SP_RUN_AS: ${{ secrets.DEV_SERVICE_PRINCIPAL_RUN_AS }}
+          GROUP_TO_MANAGE: ${{ secrets.DEV_GROUP_TO_MANAGE }}
+          PIPELINE_SA_EMAIL: ${{ secrets.DEV_PIPELINE_SA_EMAIL }}
+          END_USER_EMAIL: ${{ secrets.DEV_END_USER_EMAIL }}
+          DATAKIND_EMAIL: ${{ secrets.DEV_DATAKIND_EMAIL }}
+          SCHEMAS_PATH: ${{ secrets.DEV_CUSTOM_SCHEMAS_PATH }}
+        run: |
+          echo "Running DAB job in dev..."
+          databricks bundle deploy \
+            --profile="$PROFILE" \
+            --target=prod \
+            --var="DB_workspace=$DB_WORKSPACE" \
+            --var="service_account_executer=$SA_EXECUTER" \
+            --var="service_principal_run_as=$SP_RUN_AS" \
+            --var="datakind_group_to_manage_workflow=$GROUP_TO_MANAGE" \
+            --var="pipeline_sa_email=$PIPELINE_SA_EMAIL" \
+            --var="end_user_notification_email=$END_USER_EMAIL" \
+            --var="datakind_notification_email=$DATAKIND_EMAIL" \
+            --var="pip_install_branch=${{ github.head_ref || github.ref_name }}" \
+            --var="custom_schemas_path=$SCHEMAS_PATH"
+          databricks bundle run github_sourced_pdp_inference_pipeline --target dev
+
+  deploy-to-prod:
+    name: Deploy DAB to Staging 
+    runs-on: ubuntu-latest
+    needs: integration-test-dev
+    if: success()
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          uv venv
+          uv pip install .
+
+      - name: Set up Databricks CLI for Staging Workspace
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_STAGING_HOST }}
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_STAGING_TOKEN }}
+        run: |
+          databricks auth login --host "$DATABRICKS_HOST" --token "$DATABRICKS_TOKEN"
+          databricks auth validate
+
+      - name: Deploy to staging workspace
+        env:
+          PROFILE: ${{ secrets.STAGING_PROFILE_NAME }}
+          DB_WORKSPACE: ${{ secrets.STAGING_DB_WORKSPACE }}
+          SA_EXECUTER: ${{ secrets.STAGING_SERVICE_ACCOUNT_EXECUTER }}
+          SP_RUN_AS: ${{ secrets.STAGING_SERVICE_PRINCIPAL_RUN_AS }}
+          GROUP_TO_MANAGE: ${{ secrets.STAGING_GROUP_TO_MANAGE }}
+          PIPELINE_SA_EMAIL: ${{ secrets.STAGING_PIPELINE_SA_EMAIL }}
+          END_USER_EMAIL: ${{ secrets.STAGING_END_USER_EMAIL }}
+          DATAKIND_EMAIL: ${{ secrets.STAGING_DATAKIND_EMAIL }}
+          SCHEMAS_PATH: ${{ secrets.STAGING_CUSTOM_SCHEMAS_PATH }}
+        run: |
+          databricks bundle deploy \
+            --profile="$PROFILE" \
+            --target=prod \
+            --var="DB_workspace=$DB_WORKSPACE" \
+            --var="service_account_executer=$SA_EXECUTER" \
+            --var="service_principal_run_as=$SP_RUN_AS" \
+            --var="datakind_group_to_manage_workflow=$GROUP_TO_MANAGE" \
+            --var="pipeline_sa_email=$PIPELINE_SA_EMAIL" \
+            --var="end_user_notification_email=$END_USER_EMAIL" \
+            --var="datakind_notification_email=$DATAKIND_EMAIL" \
+            --var="pip_install_branch=${{ github.head_ref || github.ref_name }}" \
+            --var="custom_schemas_path=$SCHEMAS_PATH"
+