Developing DS Code

Author: ciaran28

data_science/src_nyc_taxi/training/__init__.py

@@ -207,8 +207,8 @@ def set_mlflow(
     ):
     if namespace.env is not None:
         params = yaml.safe_load(pathlib.Path(namespace.env).read_text())
-        experiment_name = params['ML_PIPELINE_FILES']['TRAIN_REGISTER']['PARAMETERS']['EXPERIMENT_NAME']
-        track_in_azure_ml = params['ML_PIPELINE_FILES']['TRAIN_REGISTER']['PARAMETERS']['TRACK_IN_AZURE_ML']
+        experiment_name = params['Global']['ExperimentName']
+        track_in_azure_ml = params['Global']['AMLTraking']
 
         if track_in_azure_ml:
             if track_in_azure_ml: 

mlOps/nyc_taxi/databricks_pipelines/hyper_params.yaml

@@ -1,32 +1,48 @@
-ModelOne:
-  objective": "regression"
-  "metric": "rmse"
-  "num_leaves": 25
-  "learning_rate": 0.2
-  "bagging_fraction": 0.9
-  "feature_fraction": 0.9
-  "bagging_seed": 42
-  "verbosity": -1
-  "seed": 42
+Global:
+  ExperimentName: "nyc_taxi_dbx_job"
+  AMLTraking: False
 
-ModelTwo:
-  objective": "regression"
-  "metric": "rmse"
-  "num_leaves": 27
-  "learning_rate": 0.3
-  "bagging_fraction": 0.9
-  "feature_fraction": 0.9
-  "bagging_seed": 42
-  "verbosity": -1
-  "seed": 42
-
-ModelThree:
-  objective": "regression"
-  "metric": "rmse"
-  "num_leaves": 30
-  "learning_rate": 0.4
-  "bagging_fraction": 0.9
-  "feature_fraction": 0.9
-  "bagging_seed": 42
-  "verbosity": -1
-  "seed": 42
+ModelConfigs: [
+  {
+    "ModelName": "ModelOne",
+    "ModelParams": {
+      "objective": "regression",
+      "metric": "rmse",
+      "num_leaves": 25,
+      "learning_rate": 0.2,
+      "bagging_fraction": 0.9,
+      "feature_fraction": 0.9,
+      "bagging_seed": 42,
+      "verbosity": -1,
+      "seed": 42
+    }
+  },
+  {
+    "ModelName": "ModelTwo",
+    "ModelParams": {
+      "objective": "regression",
+      "metric": "rmse",
+      "num_leaves": 27,
+      "learning_rate": 0.3,
+      "bagging_fraction": 0.9,
+      "feature_fraction": 0.9,
+      "bagging_seed": 42,
+      "verbosity": -1,
+      "seed": 42
+    }
+  },
+  {
+    "ModelName": "ModelThree",
+    "ModelParams": {
+      "objective": "regression",
+      "metric": "rmse",
+      "num_leaves": 30,
+      "learning_rate": 0.4,
+      "bagging_fraction": 0.9,
+      "feature_fraction": 0.9,
+      "bagging_seed": 42,
+      "verbosity": -1,
+      "seed": 42
+    }
+  }
+]