Developing DS Code

Author: ciaran28

experiments/notebooks/ciaran_experiments/nyc_taxi/nyc_taxi_lgbm_pyfunc_1.ipynb

@@ -2,7 +2,7 @@
 
 echo $ENVIRONMENT
 echo "Ingest JSON File"
-JSON=$( jq '.' MLOps/DevOps/Infrastructure/DBX_CICD_Deployment/Bicep_Params/$ENVIRONMENT/Bicep.parameters.json)
+JSON=$( jq '.' infrastructure/bicep/params/$ENVIRONMENT/bicep.parameters.json)
 
 TemplateParamFilePath=$( jq -r '.parameters.TemplateParamFilePath.value' <<< "$JSON")
 echo "Parm File Path: $TemplateParamFilePath"

experiments/notebooks/ciaran_experiments/nyc_taxi/test_function.py

@@ -1,5 +1,5 @@
 echo "Ingest JSON File"
-JSON=$( jq '.' mlOps/devOps/params/$ENVIRONMENT/repos.json)
+JSON=$( jq '.' infrastructure/databricks/databricks_configs/$ENVIRONMENT/repos.json)
 
 #echo "${JSON}" | jq
 

infrastructure/databricks/databricks_utils/bash/utilsCreateAzureResources.sh

@@ -6,7 +6,7 @@ echo "ENVIRONMENT: $ENVIRONMENT"
 RESOURCE_GROUP_ID=$( az group show -n $RESOURCE_GROUP_NAME --query id -o tsv )
 
 echo "Ingest JSON File"
-JSON=$( jq '.' mlOps/devOps/params/$ENVIRONMENT/rbac.json)
+JSON=$( jq '.' infrastructure/databricks/databricks_configs/$ENVIRONMENT/rbac.json)
 #echo "${JSON}" | jq
 
 for row in $(echo "${JSON}" | jq -r '.RBAC_Assignments[] | @base64'); do

infrastructure/databricks/databricks_utils/bash/utilsCreateRepoFolder.sh

@@ -7,7 +7,7 @@ REPOS_WITH_MANAGEMENT_PERMISSIONS=$(curl -X GET \
 
 
 echo "Ingest JSON File"
-JSON=$( jq '.' mlOps/devOps/params/$ENVIRONMENT/repos.json)
+JSON=$( jq '.' infrastructure/databricks/databricks_configs/$ENVIRONMENT/repos.json)
 for row in $(echo "${JSON}" | jq -r '.Repo_Configuration[] | @base64'); do
     _jq() {
         echo ${row} | base64 --decode | jq -r ${1}

infrastructure/databricks/databricks_utils/bash/utilsCreateRoleBasedAccess.sh

@@ -4,7 +4,7 @@
 echo $ENVIRONMENT
 
 echo "Ingest JSON File"
-JSON=$( jq '.' mlOps/devOps/infra/master_templates/params/$ENVIRONMENT/bicep.parameters.json)
+JSON=$( jq '.' infrastructure/bicep/params/$ENVIRONMENT/bicep.parameters.json)
 
 
 RESOURCE_GROUP_NAME=$( jq -r '.parameters.resourceGroupName.value' <<< "$JSON")

infrastructure/databricks/databricks_utils/bash/utilsRepoPull.sh

@@ -30,7 +30,7 @@ def _ingest_cluster_param_file(filename):
 
 def create_clusters():   
     ENVIRONMENT = os.environ.get("ENVIRONMENT")
-    cluster_param_file = _ingest_cluster_param_file('mlOps/devOps/params/' + ENVIRONMENT + '/clusters.json') 
+    cluster_param_file = _ingest_cluster_param_file('infrastructure/databricks/databricks_configs/' + ENVIRONMENT + '/clusters.json') 
     existing_clusters, _ = _list_existing_clusters()
     existing_clusters_name_arr = _get_cluster_names(existing_clusters)
     print(existing_clusters_name_arr)

infrastructure/databricks/databricks_utils/bash/utilsSetEnvVariables.sh

@@ -68,7 +68,7 @@ def main():
 
     ENVIRONMENT = os.environ.get("ENVIRONMENT")
 
-    file_name = 'mlOps/devOps/params/' + ENVIRONMENT + '/repos.json'
+    file_name = 'infrastructure/databricks/databricks_configs/' + ENVIRONMENT + '/repos.json'
     repo_param_file = _ingest_repo_param_file(file_name)
 
     # Extract array from Json object

infrastructure/databricks/databricks_utils/python/utils_create_cluster.py

@@ -42,7 +42,7 @@ def configureGit(gitConfig, workspaceId, databricksInstance, bearerToken, manage
 
 if __name__ == "__main__":
 
-    with open('mlOps/devOps/params/' + os.environ['ENVIRONMENT'] +'/repos.json', 'r') as f:
+    with open('infrastructure/databricks/databricks_configs/' + os.environ['ENVIRONMENT'] +'/repos.json', 'r') as f:
         json = json.load(f)
 
     gitConfigs = json['Git_Configuration']