Developing DS Code

Author: ciaran28

data_science/src_nyc_taxi/training/__init__.py

@@ -358,8 +358,6 @@ def evaluate(training_df,random_state, model):
         )
 
     evaluation_dict["r2"] = r2
-
-    #import pdb; pdb.set_trace()
 
     mlflow.log_metric(
         "r2",
@@ -398,11 +396,10 @@ def train_model_lgbm(
     # Collect data into a Pandas array for training
     data = training_df.toPandas()[features_and_label]
     train, test = train_test_split(data, random_state=123)
+    
     X_train = train.drop(["fare_amount"], axis=1)
     y_train = train.fare_amount
 
-
-
     mlflow.end_run()
     mlflow.autolog(exclusive=False)
     with mlflow.start_run():
@@ -416,29 +413,26 @@ def train_model_lgbm(
         #    label=y_test.values
         #    )
 
-        
+        num_rounds = model_params["num_rounds"]
+
         # Train a lightGBM model
         model = lgb.train(
-        #param, 
         model_params,
         train_lgb_dataset, 
         num_rounds
         )
 
-
+        mlflow.log_param("num_rounds", num_rounds)
         mlflow.log_param("local_model_file_path", model_file_path)  
 
         evaulation_dict = evaluate(
             training_df=training_df,
             random_state=123,
             model=model
             )
-
-        #import pdb; pdb.set_trace()
 
         mlflow.log_metrics(evaulation_dict)
 
-    
         fs.log_model(
             model,
             artifact_path="model_packaged",
@@ -621,7 +615,8 @@ def dbx_execute_functions():
             "feature_fraction": 0.9,
             "bagging_seed": 42,
             "verbosity": -1,
-            "seed": 42
+            "seed": 42,
+            "num_rounds": 100
         }
         )
 

infrastructure/bicep/az_templates/az_key_vault/az_key_vault.bicep

@@ -23,33 +23,6 @@ resource azKeyVault 'Microsoft.KeyVault/vaults@2021-10-01' = {
     enableSoftDelete: true
     enabledForTemplateDeployment: true
     accessPolicies: [
-        {
-          //applicationId: '<>' // Application ID of databricks SPN
-          permissions: {
-            // Give it the ability to set secrets // we can then get rid of the Key Vault Admin permission set in the main pipeline
-              // Can we do this for the main spn , the equivalent of serviceConnect1
-            secrets: [
-            'set'
-            'list'
-            'get'
-          ]
-          }
-          tenantId: subscription().tenantId
-          objectId: '<>'
-        }
-        
-        {
-        //applicationId: '<>' // Application ID of serviceConnect1
-        permissions: {
-          secrets: [
-            'set'
-            'list'
-            'get'
-          ]
-        }
-        tenantId: subscription().tenantId
-        objectId: '<>'
-      }
     ]
   }