Building MloPS + Data Science Scripts

Author: ciaran28

.github/workflows/taskDatabricks.yaml

@@ -220,7 +220,7 @@ jobs:
         name:                     Set Up DBX Environment Variables
         run: |
           DATABRICKS_TOKEN=$(az keyvault secret show --name "dbkstoken" --vault-name $AZ_KEYVAULT_NAME --query "value" -o tsv)
-          echo $DATABRICKS_TOKEN
+          #echo $DATABRICKS_TOKEN
           echo "DATABRICKS_TOKEN=$DATABRICKS_TOKEN" >> $GITHUB_ENV
 
 
@@ -235,8 +235,8 @@ jobs:
           # not the Databricks AAD Token. 
           pip3 install dbx
 
-          echo $DATABRICKS_TOKEN
-          echo $DATABRICKS_HOST
+          #echo $DATABRICKS_TOKEN
+          #echo $DATABRICKS_HOST
 
           databricks -h 
           databricks fs ls

infrastructure/databricks/databricks_utils/bash/utilsCreateDatabricksToken.sh

@@ -21,7 +21,7 @@ if [ $SECRET_EXISTS == true ]; then
                     --query "value" \
                     -o tsv )
 
-    echo "Secret Value: $DATABRICKS_TOKEN"
+    #echo "Secret Value: $DATABRICKS_TOKEN"
 
    # if [[ $DevOps_Agent == "GitHub" ]]; then
    #     echo "DATABRICKS_TOKEN=$DATABRICKS_TOKEN" >> $GITHUB_ENV

infrastructure/databricks/databricks_utils/bash/utilsCreateRoleBasedAccess.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 
-echo "Resource Group Name: $RESOURCE_GROUP_NAME"
+#echo "Resource Group Name: $RESOURCE_GROUP_NAME"
 echo "ENVIRONMENT: $ENVIRONMENT"
 RESOURCE_GROUP_ID=$( az group show -n $RESOURCE_GROUP_NAME --query id -o tsv )
 
@@ -14,7 +14,7 @@ for row in $(echo "${JSON}" | jq -r '.RBAC_Assignments[] | @base64'); do
         echo ${row} | base64 --decode | jq -r ${1}
     }
     ROLES_ARRAY="$(_jq '.roles')"
-    echo $ROLES_ARRAY
+    #echo $ROLES_ARRAY
 
     # Before: [ "Contributor", "DBX_Custom_Role", "Key Vault Administrator" ]
     # xargs trims whitespace on either side. -n removes newline characters.
@@ -30,7 +30,8 @@ for row in $(echo "${JSON}" | jq -r '.RBAC_Assignments[] | @base64'); do
         --role "$ROLE" \
         --assignee-object-id $(_jq '.roleBeneficiaryObjID') \
         --assignee-principal-type "$(_jq '.principalType')" \
-        --scope "$RESOURCE_GROUP_ID"
+        --scope "$RESOURCE_GROUP_ID" \
+        -o none
         #--scope "$(_jq '.scope')"
 
     done    

infrastructure/databricks/databricks_utils/bash/utilsSetEnvVariables.sh

@@ -17,13 +17,13 @@ DATABRICKS_INSTANCE="$(az databricks workspace list -g $RESOURCE_GROUP_NAME --qu
 WORKSPACE_ID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].id" -o tsv)
 AZ_KEYVAULT_NAME=$(az keyvault list -g $RESOURCE_GROUP_NAME --query "[].name" -o tsv)
 SUBSCRIPTION_ID=$( az account show --query id -o tsv )
-echo $SUBSCRIPTION_ID
-echo $DATABRICKS_ORDGID
-echo $WORKSPACE_ID
-echo $AZ_KEYVAULT_NAME
-echo $SUBSCRIPTION_ID
-echo $AML_WS_NAME
-echo $DATABRICKS_WS_NAME
+#echo $SUBSCRIPTION_ID
+#echo $DATABRICKS_ORDGID
+#echo $WORKSPACE_ID
+#echo $AZ_KEYVAULT_NAME
+#echo $SUBSCRIPTION_ID
+#echo $AML_WS_NAME
+#echo $DATABRICKS_WS_NAME
 #DATABRICKS_TOKEN=$(az keyvault secret show --name "dbkstoken" --vault-name $AZ_KEYVAULT_NAME --query "value" -o tsv)
 
 

infrastructure/databricks/databricks_utils/python/utils_create_cluster.py

@@ -33,7 +33,7 @@ def create_clusters():
     cluster_param_file = _ingest_cluster_param_file('infrastructure/databricks/databricks_configs/' + ENVIRONMENT + '/clusters.json') 
     existing_clusters, _ = _list_existing_clusters()
     existing_clusters_name_arr = _get_cluster_names(existing_clusters)
-    print(existing_clusters_name_arr)
+    #print(existing_clusters_name_arr)
     for cluster in cluster_param_file:
         if cluster['cluster_name'] not in existing_clusters_name_arr:
             print(f"Cluster {cluster} does not exist - Deploy.")
@@ -68,7 +68,7 @@ def _list_existing_clusters():
     status_code = response.status_code
 
     response_content = response.json()
-    print(response_content)
+    #print(response_content)
 
     if status_code != 200:
         raise Exception(status_code)

infrastructure/databricks/databricks_utils/python/utils_create_secret_scopes.py

@@ -29,11 +29,11 @@ def run_cmd(cmd):
     #May Need To Rmove shell=True
     process = subprocess.run(cmd, stdout=subprocess.PIPE)
     output = process.stdout.decode().split('\n')
-    print(output)
+    #print(output)
     output = [line.strip('\n').strip('\r') for line in output]
 
 
-    print(f"Return Code: {process.returncode}")
+    #print(f"Return Code: {process.returncode}")
     if process.returncode != 0:
         raise RuntimeError('\n'.join(output))
     return output
@@ -63,11 +63,11 @@ def create_secret_scopes(scope_name=str, initial_manage_principal=str):
         'https://' + DATABRICKS_INSTANCE + '/api/2.0/secrets/scopes/create', headers=DBRKS_REQ_HEADERS, json=postjson
     )
 
-    print(response.status_code)
+    #print(response.status_code)
     #if response.status_code != 200:
     #    raise Exception(response.text)
 
-    print(response.json())
+    #print(response.json())
 
 def insert_secret(secret_value=str, scope_name=str, key=str):
     """
@@ -82,18 +82,18 @@ def insert_secret(secret_value=str, scope_name=str, key=str):
     response = requests.post(
         'https://' + DATABRICKS_INSTANCE + '/api/2.0/secrets/put', headers=DBRKS_REQ_HEADERS, json=postjson
     )
-    print(response.status_code)
+    #print(response.status_code)
     if response.status_code != 200:
         raise Exception(response.text)
 
-    print(response.json())
+    #print(response.json())
 
 
 if __name__ == '__main__':
     app_insight_name = get_app_insight_name()[0]
-    print(app_insight_name)
+    #print(app_insight_name)
     app_insight_key = get_app_insight_key(app_insight_name)[0]
-    print(app_insight_key)
+    #print(app_insight_key)
 
 
     # Create Secret Scopes

infrastructure/databricks/databricks_utils/python/utils_git_configuration.py

@@ -23,19 +23,19 @@ def configureGit(gitConfig, workspaceId, databricksInstance, bearerToken, manage
         }
 
     gitConfig.update(newData)
-    print(gitConfig)
-    print(DBRKS_REQ_HEADERS)
+    #print(gitConfig)
+    #print(DBRKS_REQ_HEADERS)
 
     response = requests.post('https://' + databricksInstance + '/api/2.0/git-credentials', headers=DBRKS_REQ_HEADERS, json=gitConfig)
-    print(response)
+    #print(response)
     #print(response.json())
 
     if response.status_code != 200:
 
         response = requests.get('https://' + databricksInstance + '/api/2.0/git-credentials', headers=DBRKS_REQ_HEADERS)
-        print(response.json())
+        #print(response.json())
         credential = response.json()["credentials"][0]["credential_id"]
-        print(f"Credential is {credential}")
+        #print(f"Credential is {credential}")
         response = requests.patch('https://' + databricksInstance + '/api/2.0/git-credentials/'+ str(credential), headers=DBRKS_REQ_HEADERS, json=gitConfig)
 
     #print(response.json())
@@ -47,12 +47,12 @@ def configureGit(gitConfig, workspaceId, databricksInstance, bearerToken, manage
 
     gitConfigs = json['Git_Configuration']
 
-    print(os.environ['WORKSPACE_ID'])
-    print(os.environ['DATABRICKS_INSTANCE'])
-    print(os.environ['DATABRICKS_AAD_TOKEN'])
-    print(os.environ['DATABRICKS_MANAGEMENT_TOKEN'])
-    print(os.environ['PAT_GITHUB'])
-    print(os.environ['ENVIRONMENT'])
+    #print(os.environ['WORKSPACE_ID'])
+    #print(os.environ['DATABRICKS_INSTANCE'])
+    #print(os.environ['DATABRICKS_AAD_TOKEN'])
+    #print(os.environ['DATABRICKS_MANAGEMENT_TOKEN'])
+    #print(os.environ['PAT_GITHUB'])
+    #print(os.environ['ENVIRONMENT'])
     for gitConfig in gitConfigs:
         response = configureGit(gitConfig=gitConfig, 
                                 workspaceId=os.environ['WORKSPACE_ID'], 

infrastructure/databricks/databricks_utils/python/utils_repo_pull.py

@@ -84,8 +84,8 @@ def update_repo(repo_id, update_branch):
     if response.status_code != 200:
         raise Exception(response.content)
     else:
-        print(f"Status Code: {response.status_code}")
-        print(response.json())
+        #print(f"Status Code: {response.status_code}")
+        #print(response.json())
         return response.status_code
 
 

mlOps/nyc_taxi/aml_pipelines/v1/nyc_pipeline.py

@@ -31,17 +31,17 @@
     'Content-Type': 'application/json'
 }
 
-print(DATABRICKS_COMPUTE_NAME)
-print(WORKSPACE_ID)
-print(RESOURCE_GROUP_NAME)
-print(DATABRICKS_AAD_TOKEN)
-print(DATABRICKS_MANAGEMENT_TOKEN)
-print(DATABRICKS_INSTANCE)
-print(SUBSCRIPTION_ID)
-print(ARM_CLIENT_SECRET)
-print(ARM_TENANT_ID)
-print(ARM_CLIENT_ID)
-print(AML_WS_NAME)
+#print(DATABRICKS_COMPUTE_NAME)
+#print(WORKSPACE_ID)
+#print(RESOURCE_GROUP_NAME)
+#print(DATABRICKS_AAD_TOKEN)
+#print(DATABRICKS_MANAGEMENT_TOKEN)
+#print(DATABRICKS_INSTANCE)
+#print(SUBSCRIPTION_ID)
+#print(ARM_CLIENT_SECRET)
+#print(ARM_TENANT_ID)
+#print(ARM_CLIENT_ID)
+#print(AML_WS_NAME)
 
 class GetClusterID():
     def __init__(self, cluster_name):
@@ -103,17 +103,17 @@ def create_pipeline_structure(databricks_compute, ws, cluster_id):
         auth=svc_pr
     )
 
-    print(f" AML Workspace Properties: {ws} ")
+    #print(f" AML Workspace Properties: {ws} ")
 
     try:
         databricks_compute = DatabricksCompute(workspace=ws, name=DATABRICKS_COMPUTE_NAME)
         print('Compute target {} already exists'.format(DATABRICKS_COMPUTE_NAME))
     except ComputeTargetException:
-        print('Compute not found, will use below parameters to attach new one')
-        print('db_compute_name {}'.format(DATABRICKS_COMPUTE_NAME))
-        print('db_resource_group {}'.format(RESOURCE_GROUP_NAME))
-        print('db_workspace_name {}'.format(DATABRICKS_WS_NAME))
-        print('db_access_token {}'.format(DATABRICKS_AAD_TOKEN))
+        #print('Compute not found, will use below parameters to attach new one')
+        #print('db_compute_name {}'.format(DATABRICKS_COMPUTE_NAME))
+        #print('db_resource_group {}'.format(RESOURCE_GROUP_NAME))
+        #print('db_workspace_name {}'.format(DATABRICKS_WS_NAME))
+        #print('db_access_token {}'.format(DATABRICKS_AAD_TOKEN))
 
         config = DatabricksCompute.attach_configuration(
             resource_group = RESOURCE_GROUP_NAME,