Skip to content

Commit 14825db

Browse files
ciaran28ciaran28
authored
Merge pull request #87 from ciaran28/main
Bug Fixes + Testing Framework Development + Refactoring
2 parents 9582886 + 2f186b7 commit 14825db

File tree

47 files changed

+1567
-784
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1567
-784
lines changed

‎.dbx/lock.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
{
2-
"context_id": "4958908691327254224"
1+
{
2+
"context_id": "6505588973318184615"
33
}

‎.dbx/project.json‎

Lines changed: 19 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,20 @@
1-
{
2-
"environments": {
3-
"sandbox": {
4-
"profile": "sandbox",
5-
"workspace_dir": "/Shared/cicd_workflows/sandbox",
6-
"storage_type": "mlflow",
7-
"properties": {
8-
"workspace_directory": "/Shared/cicd_workflows/sandbox",
9-
"artifact_location": "dbfs:/Shared/cicd_workflows/sandbox"
10-
}
11-
},
12-
"development": {
13-
"profile": "development",
14-
"workspace_dir": "/Shared/development",
15-
"artifact_location": "dbfs:/Shared/cicd_workflows/development"
16-
},
17-
"uat": {
18-
"profile": "uat",
19-
"workspace_dir": "/Shared/uat/uat",
20-
"artifact_location": "dbfs:/Shared/cicd_workflows/uat"
21-
},
22-
"production": {
23-
"profile": "production",
24-
"workspace_dir": "/Shared/production",
25-
"artifact_location": "dbfs:/Shared/cicd_workflows/production"
26-
},
27-
"ciaran_sandbox": {
28-
"profile": "ciaran_sandbox",
29-
"workspace_dir": "/Shared/ciaran_sandbox",
30-
"artifact_location": "dbfs:/Shared/cicd_workflows/ciaran_sandbox"
31-
},
32-
"default": {
33-
"profile": "default",
34-
"storage_type": "mlflow",
35-
"properties": {
36-
"workspace_directory": "/Shared/cicd_workflows/default",
37-
"artifact_location": "dbfs:/Shared/cicd_workflows/default"
38-
}
39-
}
40-
},
41-
"inplace_jinja_support": true,
42-
"failsafe_cluster_reuse_with_assets": false,
43-
"context_based_upload_for_execute": false
1+
{
2+
"environments": {
3+
"default": {
4+
"profile": "DEFAULT",
5+
"storage_type": "mlflow",
6+
"properties": {
7+
"workspace_directory": "/Shared/cicd_dbx_workflows",
8+
"artifact_location": "dbfs:/Shared/cicd_dbx_workflows"
9+
}
10+
},
11+
"ciaran_sandbox": {
12+
"profile": "ciaran_sandbox",
13+
"workspace_dir": "/Shared/ciaran_sandbox",
14+
"artifact_location": "dbfs:/Shared/cicd_workflows/ciaran_sandbox"
15+
}
16+
},
17+
"inplace_jinja_support": true,
18+
"failsafe_cluster_reuse_with_assets": false,
19+
"context_based_upload_for_execute": false
4420
}

‎.github/workflows/onRelease.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ jobs:
2323
- run: |
2424
echo "Insert Continuous Integration Tests"
2525
26+
# IMPORTANT: The testing framework is not yet implemented, and therefore still under development.
27+
28+
cd mlOps/devOps/utils
29+
30+
python -m pytest -v
31+
2632
prApproved_CD_Development:
2733
if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && contains(github.head_ref, 'feature') && github.base_ref == 'main'
2834
uses: ./.github/workflows/taskDatabricks.yaml

‎.github/workflows/taskDatabricks.yaml‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ jobs:
3131
build:
3232
name: ${{ inputs.ENVIRONMENT }} Environment Deployment
3333
runs-on: ubuntu-latest
34+
#runs-on: self-hosted
3435
environment: ${{ inputs.ENVIRONMENT }}
3536
steps:
3637
- name: Checkout
@@ -44,23 +45,31 @@ jobs:
4445
- name: Setup Python
4546
uses: actions/setup-python@v4
4647
with:
47-
python-version: 3.8
48+
python-version: "3.8"
4849

4950
#################################/
5051
## Dependencies/ Install
5152
#################################/
5253
- shell: bash
5354
name: Tests and Dependency Install
5455
run: |
55-
56-
sudo apt update && sudo apt upgrade && sudo apt install jq -y
56+
5757
pip3 install --upgrade pip
5858
59+
#sudo apt update && sudo apt upgrade && sudo apt install jq -y
60+
61+
5962
az extension add -n azure-cli-ml
6063
6164
# Do a python pip install so that modules can be found from python tasks
6265
python -m pip install requests
66+
python -m pip install python-dotenv
67+
68+
6369
70+
71+
72+
6473
pip3 install --upgrade azure-cli
6574
pip3 install azure-mgmt-storage==21.0.0
6675
#pip3 install azure-databricks-sdk-python
@@ -105,7 +114,7 @@ jobs:
105114
## Generate AAD Tokens
106115
#################################/
107116
- name: Generate AAD Tokens
108-
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utilsCreateAuthenticationTokens.py
117+
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utils_create_authentication_tokens.py
109118
env:
110119
ENVIRONMENT: ${{ inputs.ENVIRONMENT }}
111120
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
@@ -146,23 +155,23 @@ jobs:
146155
#################################/
147156

148157
- name: Git Config
149-
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utilsGitConfiguration.py
158+
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utils_git_configuration.py
150159
env:
151160
PAT_GITHUB: ${{ secrets.PAT_GITHUB }}
152161

153162
#################################/
154163
## Create Repos
155164
#################################/
156165
- name: Create Databricks Repos
157-
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utilsCreateRepoFolder.py
166+
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utils_create_repo_folder.py
158167
env:
159168
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
160169

161170
#################################/
162171
## Secret Scopes
163172
#################################/
164173
- name: Save Databricks SP Credentials Within Secret Scopes
165-
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utilsCreateSecretScopes.py
174+
run: ${{ inputs.SCRIPT_LANGUAGE }} mlOps/devOps/utils/${{ inputs.SCRIPT_LANGUAGE }}/utils_create_secret_scopes.py
166175
env:
167176
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
168177
ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
@@ -235,10 +244,11 @@ jobs:
235244
databricks -h
236245
databricks fs ls
237246
238-
dbx configure --enable-inplace-jinja-support --profile $ENVIRONMENT
247+
dbx configure --enable-inplace-jinja-support --profile default
239248
240-
dbx deploy NYC_TAXI --no-package --deployment-file mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow.yaml \
241-
-e $ENVIRONMENT --jinja-variables-file=mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow_params.yaml
249+
# -e replaced with "default" instead of $ENVIRONMENT
250+
dbx deploy --workflows JOB_WORKFLOW_NYC_TAXI,TRAIN_REGISTER --no-package --deployment-file mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow.yaml \
251+
-e default --jinja-variables-file=mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow_params.yaml
242252
243253
env:
244254
ENVIRONMENT: ${{ inputs.ENVIRONMENT }}

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ mlOps/devOps/params/deprecated
77
.dbx/sync
88
localdev
99
azureDevOps
10+
venv_dbx_connect
11+
venv_dbx
12+
.env
13+
.linux_venv
14+
15+
1016

1117

1218

‎mlOps/dataOps/nyc_taxi/data_prep.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,7 @@ def write_to_table(df, database, table):
138138

139139
# COMMAND ----------
140140

141-
# MAGIC %sql
142-
# MAGIC CREATE DATABASE IF NOT EXISTS feature_store_taxi_example;
141+
spark.sql("CREATE DATABASE IF NOT EXISTS feature_store_taxi_example;")
143142

144143
# COMMAND ----------
145144

‎mlOps/devOps/params/development/rbac.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"roles": [
55
"Key Vault Administrator"
66
],
7-
"roleBeneficiaryObjID": "888a0304-df2d-4e40-b0e8-48ea893539ef",
7+
"roleBeneficiaryObjID": "121b9734-5ce8-437d-85dc-7603f778f771",
88
"Description": "Your Object ID",
99
"principalType": "User"
1010
},
@@ -14,7 +14,7 @@
1414
"Key Vault Administrator",
1515
"Storage Blob Data Contributor"
1616
],
17-
"roleBeneficiaryObjID": "0e3b4dff-0dac-4473-b5ba-ca3c5934ef79",
17+
"roleBeneficiaryObjID": "6c74b84d-e229-4660-b51d-b8f7dae16c67",
1818
"Description": "Databricks SPN",
1919
"principalType": "ServicePrincipal"
2020
}

‎mlOps/devOps/params/development/repos.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
{
1010
"url": "https://github.com/ciaran28/dstoolkit-mlops-databricks",
1111
"provider": "gitHub",
12-
"path": "DevelopmentFolder"
12+
"path": "DevelopmentFolder",
13+
"branch": "main"
1314
}
1415
]
1516
}

‎mlOps/devOps/params/production/rbac.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"roles": [
55
"Key Vault Administrator"
66
],
7-
"roleBeneficiaryObjID": "888a0304-df2d-4e40-b0e8-48ea893539ef",
7+
"roleBeneficiaryObjID": "121b9734-5ce8-437d-85dc-7603f778f771",
88
"Description": "Your Object ID",
99
"principalType": "User"
1010
},
@@ -14,7 +14,7 @@
1414
"Key Vault Administrator",
1515
"Storage Blob Data Contributor"
1616
],
17-
"roleBeneficiaryObjID": "0e3b4dff-0dac-4473-b5ba-ca3c5934ef79",
17+
"roleBeneficiaryObjID": "6c74b84d-e229-4660-b51d-b8f7dae16c67",
1818
"Description": "Databricks SPN",
1919
"principalType": "ServicePrincipal"
2020
}

‎mlOps/devOps/params/production/repos.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
{
1010
"url": "https://github.com/ciaran28/dstoolkit-mlops-databricks",
1111
"provider": "gitHub",
12-
"path": "ProductionFolder"
12+
"path": "ProductionFolder",
13+
"branch": "main"
1314
}
1415
]
1516
}

0 commit comments

Comments
 (0)