Developing DS Code

Author: ciaran28

data_science/src_nyc_taxi/registration/__init__.py

@@ -1,6 +1,6 @@
 # Databricks notebook source
 
-
+from databricks.sdk.runtime import *
 from databricks import feature_store
 from pyspark.sql.types import *
 from pyspark.sql.functions import *

data_science/src_nyc_taxi/training/__init__.py

@@ -8,7 +8,7 @@
 
 #dbutils.library.restartPython()
 # COMMAND ----------
-
+from databricks.sdk.runtime import *
 import os
 from sklearn import metrics
 import joblib
@@ -38,6 +38,7 @@
 from ds_utils import * # Custom Package
 import base64
 import datetime
+from pyspark.dbutils import DBUtils
 
 
 # COMMAND ----------
@@ -114,6 +115,10 @@ def get_secret(scope: str, secret_name: str) -> str:
 
     if os.environ.get(secret_name):
         return os.environ.get(secret_name)
+    
+    from pyspark.dbutils import DBUtils
+    spark = SparkSession.builder.getOrCreate()
+    dbutils = DBUtils(spark)
     return dbutils.secrets.get(scope=scope, key=secret_name)
 
 def databricks_sdk_secrets(scope, secret_name):

experiments/notebooks/ciaran_experiments/nyc_taxi/nyc_taxi_lgbm_1.py

@@ -1,16 +1,11 @@
 # Databricks notebook source
 
-#from featurization import run_feature_store_refresh
-%run /Users/admin@mngenvmcap557827.onmicrosoft.com/.ide/dev/data_science/src_nyc_taxi/featurization/__init__
-# COMMAND ----------
-
+from featurization import run_feature_store_refresh
 run_feature_store_refresh()
 
 # COMMAND ----------
 from training import run_training
-%run /Users/admin@mngenvmcap557827.onmicrosoft.com/.ide/dev3/data_science/src_nyc_taxi/featurization/__init__
-# COMMAND ----------
-
+from databricks.sdk.runtime import *
 run_training(
     experiment_name = "ciaran_experiment_nyc_taxi",
     model_name = "taxi_example_fare_packaged",
@@ -26,5 +21,9 @@
         "seed": 42
     }
 )
-
-# COMMAND ----------
+# COMMAND ----------
+from databricks.sdk.runtime import *
+from registration import run_registration
+run_registration(
+    model_name = "taxi_example_fare_packaged"
+)

infrastructure/bicep/params/development/bicep.parameters.json

@@ -3,10 +3,10 @@
   "contentVersion": "1.0.0.0",
   "parameters": {
     "TemplateParamFilePath": {
-      "value": "mlOps/devOps/infra/master_templates/params/development/bicep.parameters.json"
+      "value": "infrastructure/bicep/params/development/bicep.parameters.json"
     },
     "TemplateFilePath": {
-      "value": "mlOps/devOps/infra/master_templates/main.bicep"
+      "value": "infrastructure/bicep/main.bicep"
     },
     "AZURE_DATABRICKS_APP_ID": {
       "value": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"

infrastructure/bicep/params/production/bicep.parameters.json

@@ -3,10 +3,10 @@
     "contentVersion": "1.0.0.0",
     "parameters": {
       "TemplateParamFilePath": {
-        "value": "mlOps/devOps/infra/master_templates/params/production/bicep.parameters.json"
+        "value": "infrastructure/bicep/params/production/bicep.parameters.json"
       },
       "TemplateFilePath": {
-        "value": "mlOps/devOps/infra/master_templates/main.bicep"
+        "value": "infrastructure/bicep/main.bicep"
       },
       "AZURE_DATABRICKS_APP_ID": {
         "value": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"

infrastructure/bicep/params/sandbox/bicep.parameters.json

@@ -3,10 +3,10 @@
   "contentVersion": "1.0.0.0",
   "parameters": {
     "TemplateParamFilePath": {
-      "value": "mlOps/devOps/infra/master_templates/params/sandbox/bicep.parameters.json"
+      "value": "infrastructure/bicep/params/sandbox/bicep.parameters.json"
     },
     "TemplateFilePath": {
-      "value": "mlOps/devOps/infra/master_templates/main.bicep"
+      "value": "infrastructure/bicep/main.bicep"
     },
     "AZURE_DATABRICKS_APP_ID": {
       "value": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"

infrastructure/bicep/params/uat/bicep.parameters.json

@@ -3,10 +3,10 @@
     "contentVersion": "1.0.0.0",
     "parameters": {
       "TemplateParamFilePath": {
-        "value": "mlOps/devOps/infra/master_templates/params/uat/bicep.parameters.json"
+        "value": "infrastructure/bicep/params/uat/bicep.parameters.json"
       },
       "TemplateFilePath": {
-        "value": "mlOps/devOps/infra/master_templates/main.bicep"
+        "value": "infrastructure/bicep/main.bicep"
       },
       "AZURE_DATABRICKS_APP_ID": {
         "value": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"