Developing DS Code

Author: ciaran28

data_science/src_nyc_taxi/registration/__init__.py

@@ -109,13 +109,6 @@ def evaluation(
         # Get the latest model version in the production stage
 
         mlflow_client = MlflowClient()
-        #latest_production_version = mlflow_client.get_latest_versions(
-        #    name=model_name,
-        #    stages=[production_stage],
-        #    order_by=['creation_time desc'],
-        #    max_results=1
-        #)[0].version
-
 
         latest_production_version = mlflow_client.get_latest_versions(
             name=model_name,
@@ -126,9 +119,10 @@ def evaluation(
 
 
         # Promote Latest Model To Production
+        latest_model_version = get_latest_model_version(model_name)
         mlflow_client.transition_model_version_stage(
                 name=model_name,
-                version=latest_production_version,
+                version=latest_model_version,
                 stage="production",
                 archive_existing_versions = True
         )

experiments/notebooks/ciaran_experiments/nyc_taxi/nyc_taxi_lgbm_1.py

@@ -4,16 +4,16 @@
 run_feature_store_refresh()
 
 # COMMAND ----------
-from training import run_training
-from databricks.sdk.runtime import *
+from training import run_training 
+
 run_training(
     experiment_name = "ciaran_experiment_nyc_taxi",
     model_name = "taxi_example_fare_packaged",
     model_params = {
         "objective": "regression",
         "metric": "rmse",
         "num_leaves": 32,
-        "learning_rate": 0.1,
+        "learning_rate": 0.2,
         "bagging_fraction": 0.9,
         "feature_fraction": 0.9,
         "bagging_seed": 42,
@@ -22,7 +22,6 @@
     }
 )
 # COMMAND ----------
-from databricks.sdk.runtime import *
 from registration import run_registration
 run_registration(
     model_name = "taxi_example_fare_packaged"

infrastructure/bicep/main.bicep

@@ -33,7 +33,7 @@ resource azResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = {
 //                                                                  KEY VAULT - SELECT KV                                                                                //
 // ################################################################################################################################################################//
 
-module azKeyVault '../az_templates/az_key_vault/az_key_vault.bicep' = {
+module azKeyVault 'az_templates/az_key_vault/az_key_vault.bicep' = {
   dependsOn: [
     azResourceGroup
 
@@ -50,7 +50,7 @@ module azKeyVault '../az_templates/az_key_vault/az_key_vault.bicep' = {
 //                                                                       Module for Create Azure Data Lake Storage
 // RBAC is assigned -> azDatabricks given access to Storage 
 // ################################################################################################################################################################//
-module azDataLake '../az_templates/az_data_lake/az_data_lake.bicep' =  {
+module azDataLake 'az_templates/az_data_lake/az_data_lake.bicep' =  {
   dependsOn: [
     azResourceGroup
   ]
@@ -71,7 +71,7 @@ module azDataLake '../az_templates/az_data_lake/az_data_lake.bicep' =  {
 }
 
 
-module logAnalytics '../az_templates/az_app_insights/az_app_insights.bicep'= {
+module logAnalytics 'az_templates/az_app_insights/az_app_insights.bicep' = {
   dependsOn: [
     azResourceGroup
   ]
@@ -89,7 +89,7 @@ module logAnalytics '../az_templates/az_app_insights/az_app_insights.bicep'= {
 //                                                                       Module for Creating Azure Machine Learning Workspace
 // Outputs AzDatabricks Workspace ID, which is used when Assigning RBACs.
 // ################################################################################################################################################################//
-module azMachineLearning '../az_templates/az_machine_learning/az_machine_learning.bicep' =  {
+module azMachineLearning 'az_templates/az_machine_learning/az_machine_learning.bicep' =  {
   dependsOn: [
     logAnalytics
     azDataLake
@@ -114,7 +114,7 @@ module azMachineLearning '../az_templates/az_machine_learning/az_machine_learnin
 //                                                                       Module for Creating Azure Databricks Workspace
 // Outputs AzDatabricks Workspace ID, which is used when Assigning RBACs
 // ################################################################################################################################################################//
-module azDatabricks '../az_templates/az_databricks/az_databricks.bicep' =  {
+module azDatabricks 'az_templates/az_databricks/az_databricks.bicep' =  {
   dependsOn: [
     azMachineLearning
   ]

infrastructure/databricks/databricks_utils/python/utils_create_azure_resources.py

@@ -9,7 +9,7 @@
 
 class LoadJson():
     def __init__(self):
-        self.json_file = 'mlOps/devOps/infra/master_templates/params/' + ENVIRONMENT + '/bicep.parameters.json'
+        self.json_file = 'infrastructure/bicep/params/' + ENVIRONMENT + '/bicep.parameters.json'
 
     def load_json(self):
         with open(self.json_file, 'r') as f: