Merge pull request #86 from ciaran28/main

Bug Fix

Author: ciaran28

.github/workflows/taskDatabricks.yaml

@@ -26,7 +26,7 @@ on:
         required: true
       RELEASE_BRANCH:
         required: false
-        
+
 jobs:
   build:
     name:                         ${{ inputs.ENVIRONMENT }} Environment Deployment
@@ -237,7 +237,7 @@ jobs:
 
           dbx configure --enable-inplace-jinja-support --profile $ENVIRONMENT
         
-          dbx deploy NYC_TAXI --deployment-file mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow.yaml \
+          dbx deploy NYC_TAXI --no-package --deployment-file mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow.yaml \
           -e $ENVIRONMENT --jinja-variables-file=mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow_params.yaml
 
         env:

.gitignore

@@ -13,3 +13,4 @@ azureDevOps
 
 
 
+

dist/helperfunctions-0.0.1-py3-none-any.whl

@@ -23,43 +23,6 @@
 from pytz import timezone
 from helperFunctions.helperFunction import *
 
-
-# COMMAND ----------
-
-args = dbutils.notebook.entry_point.getCurrentBindings()
-print(args)
-
-# COMMAND ----------
-
-#experiment_id = dbutils.widgets.get("--AZUREML_EXPERIMENT_ID")
-#print(experiment_id)
-
-# COMMAND ----------
-
-# Ingest Parameters Files
-
-# COMMAND ----------
-
-p = ArgumentParser()
-p.add_argument("--env", required=False, type=str)
-namespace = p.parse_known_args(sys.argv[1:])[0]
-display(namespace)
-
-
-if namespace.env is not None:
-    display(namespace.env)
-    params = yaml.safe_load(pathlib.Path(namespace.env).read_text())
-    display(params)
-    experiment_name = params['ML_PIPELINE_FILES']['FEATURE_ENGINEERING']['PARAMETERS']['EXPERIMENT_NAME']
-    display(experiment_name)
-    mlflow.set_experiment(experiment_name=experiment_name) 
-
-else:
-    display("Set The Parameters Manually, As We Are Deploying From UI")
-    mlflow.set_experiment("/Shared/dbxDevelopment") 
-
-
-
 # COMMAND ----------
 
 #Inggest Data

mlOps/modelOps/data_science/nyc_taxi/feature_eng.py

@@ -69,6 +69,6 @@ sklearn_pandas==2.2.0
 azureml-sdk==1.50.0
 uszipcode
 lightgbm
-azureml-sdk[databricks]
+azureml-sdk[databricks]==1.50.0
 
 

mlOps/modelOps/data_science/nyc_taxi/pyWheels/Helper_Functions/requirements.txt

@@ -45,8 +45,7 @@
 
 # COMMAND ----------
 
-%md
-## Ingest Args (If Triggered From Pipeline)
+# MAGIC %md ## Ingest Args (If Triggered From Pipeline)
 
 # COMMAND ----------
 
@@ -57,9 +56,7 @@
 
 # COMMAND ----------
 
-%md
-
-## Set Azure ML Configs
+# MAGIC %md ## Set Azure ML Configs
 
 # COMMAND ----------
 
@@ -91,9 +88,7 @@
 
 # COMMAND ----------
 
-%md
-
-## Set MLFlow Tracking Server
+# MAGIC %md ## Set MLFlow Tracking Server
 
 # COMMAND ----------
 
@@ -102,8 +97,8 @@
 
 if namespace.env is not None:
     params = yaml.safe_load(pathlib.Path(namespace.env).read_text())
-    experiment_name = params['ML_PIPELINE_FILES']['MODEL_TRAINING']['PARAMETERS']['EXPERIMENT_NAME']
-    track_in_azure_ml = params['ML_PIPELINE_FILES']['MODEL_TRAINING']['PARAMETERS']['TRACK_IN_AZURE_ML']
+    experiment_name = params['ML_PIPELINE_FILES']['TRAIN_REGISTER']['PARAMETERS']['EXPERIMENT_NAME']
+    track_in_azure_ml = params['ML_PIPELINE_FILES']['TRAIN_REGISTER']['PARAMETERS']['TRACK_IN_AZURE_ML']
 
     if track_in_azure_ml:
         if track_in_azure_ml: 
@@ -183,7 +178,7 @@
 
 # COMMAND ----------
 
-from sklearn.metrics import mean_squared_log_error
+from sklearn import metrics
 
 features_and_label = training_df.columns
 
@@ -199,8 +194,6 @@
 
 import joblib
 
-joblib.dump(model, open(model_file_path1,'wb'))      #Save The Model 
-
 mlflow.end_run()
 mlflow.autolog(exclusive=False)
 with mlflow.start_run():
@@ -221,6 +214,8 @@
     param, train_lgb_dataset, num_rounds
     )
 
+    joblib.dump(model, open(model_file_path1,'wb'))      #Save The Model 
+
     expected_y  = y_test
     predicted_y = model.predict(X_test)
 

mlOps/modelOps/data_science/nyc_taxi/train_register.py

@@ -29,10 +29,20 @@ build:
   no_build: true
 
 environments:
+  
   sandbox:
     workflows:
       - name: NYC_TAXI     
         tasks:
+
+          - task_key: "Data_Ingestion_And_Prep"
+            <<: *dev-cluster-config
+            spark_python_task:
+              python_file: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['FILE_PATH']}}"
+            libraries: [ 
+              whl: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['WHL_PATH']}}" 
+            ]
+
           - task_key: "Feature_Engineering"
             <<: *dev-cluster-config
             spark_python_task:
@@ -41,6 +51,8 @@ environments:
             libraries: [ 
               whl: "{{var['ML_PIPELINE_FILES']['FEATURE_ENGINEERING']['WHL_PATH']}}" 
             ]
+            depends_on:
+              - task_key: "Data_Ingestion_And_Prep"
 
           - task_key: "Train_Register"
             <<: *dev-cluster-config
@@ -68,6 +80,15 @@ environments:
     workflows:
       - name: NYC_TAXI     
         tasks:
+
+          - task_key: "Data_Ingestion_And_Prep"
+            <<: *dev-cluster-config
+            spark_python_task:
+              python_file: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['FILE_PATH']}}"
+            libraries: [ 
+              whl: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['WHL_PATH']}}" 
+            ]
+
           - task_key: "Feature_Engineering"
             <<: *dev-cluster-config
             spark_python_task:
@@ -76,6 +97,8 @@ environments:
             libraries: [ 
               whl: "{{var['ML_PIPELINE_FILES']['FEATURE_ENGINEERING']['WHL_PATH']}}" 
             ]
+            depends_on:
+              - task_key: "Data_Ingestion_And_Prep"
 
           - task_key: "Train_Register"
             <<: *dev-cluster-config
@@ -102,6 +125,15 @@ environments:
     workflows:
       - name: NYC_TAXI     
         tasks:
+
+          - task_key: "Data_Ingestion_And_Prep"
+            <<: *dev-cluster-config
+            spark_python_task:
+              python_file: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['FILE_PATH']}}"
+            libraries: [ 
+              whl: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['WHL_PATH']}}" 
+            ]
+
           - task_key: "Feature_Engineering"
             <<: *dev-cluster-config
             spark_python_task:
@@ -110,6 +142,8 @@ environments:
             libraries: [ 
               whl: "{{var['ML_PIPELINE_FILES']['FEATURE_ENGINEERING']['WHL_PATH']}}" 
             ]
+            depends_on:
+              - task_key: "Data_Ingestion_And_Prep"
 
           - task_key: "Train_Register"
             <<: *dev-cluster-config
@@ -137,6 +171,15 @@ environments:
     workflows:
       - name: NYC_TAXI     
         tasks:
+
+          - task_key: "Data_Ingestion_And_Prep"
+            <<: *dev-cluster-config
+            spark_python_task:
+              python_file: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['FILE_PATH']}}"
+            libraries: [ 
+              whl: "{{var['ML_PIPELINE_FILES']['DATA_INGEST_PREP']['WHL_PATH']}}" 
+            ]
+
           - task_key: "Feature_Engineering"
             <<: *dev-cluster-config
             spark_python_task:
@@ -145,6 +188,8 @@ environments:
             libraries: [ 
               whl: "{{var['ML_PIPELINE_FILES']['FEATURE_ENGINEERING']['WHL_PATH']}}" 
             ]
+            depends_on:
+              - task_key: "Data_Ingestion_And_Prep"
 
           - task_key: "Train_Register"
             <<: *dev-cluster-config

mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow.yaml

@@ -1,4 +1,9 @@
 ML_PIPELINE_FILES:
+
+    DATA_INGEST_PREP:
+        FILE_PATH: 'file://mlOps/dataOps/nyc_taxi/data_prep.py'
+        WHL_PATH: 'file://mlOps/modelOps/data_science/nyc_taxi/pyWheels/Helper_Functions/dist/helperfunctions-0.0.1-py3-none-any.whl'
+
     FEATURE_ENGINEERING: 
         FILE_PATH: 'file://mlOps/modelOps/data_science/nyc_taxi/feature_eng.py'
         WHL_PATH: 'file://mlOps/modelOps/data_science/nyc_taxi/pyWheels/Helper_Functions/dist/helperfunctions-0.0.1-py3-none-any.whl'
@@ -7,6 +12,7 @@ ML_PIPELINE_FILES:
             FILE: 'file:fuse://mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow_params.yaml'
             EXPERIMENT_NAME: 'dbx_workflow_fe'
             TRACK_IN_AZURE_ML: True
+
     TRAIN_REGISTER:
         FILE_PATH: 'file://mlOps/modelOps/data_science/nyc_taxi/train_register.py'
         WHL_PATH: 'file://mlOps/modelOps/data_science/nyc_taxi/pyWheels/Helper_Functions/dist/helperfunctions-0.0.1-py3-none-any.whl'
@@ -15,6 +21,7 @@ ML_PIPELINE_FILES:
             FILE: 'file:fuse://mlOps/modelOps/ml_pipelines/az_databricks/cicd/workflow_params.yaml'
             EXPERIMENT_NAME: 'dbx_workflow_train'
             TRACK_IN_AZURE_ML: True
+
     MODEL_INFERENCE:
         FILE_PATH: 'file://mlOps/modelOps/data_science/nyc_taxi/score.py'
         WHL_PATH: 'file://mlOps/modelOps/data_science/nyc_taxi/pyWheels/Helper_Functions/dist/helperfunctions-0.0.1-py3-none-any.whl'