Change folder structure to support several training pipelines and models (#147)

* change folder structure

* rename variables file

* fix linting

* documentation fix

* return model name due to a bug in scoring

* make PR build working

Author: sbaidachni

.env.example

@@ -30,7 +30,7 @@ TRAINING_PIPELINE_NAME = 'Training Pipeline'
 MODEL_PATH = ''
 EVALUATE_SCRIPT_PATH = 'evaluate/evaluate_model.py'
 REGISTER_SCRIPT_PATH = 'register/register_model.py'
-SOURCES_DIR_TRAIN = 'code'
+SOURCES_DIR_TRAIN = 'diabetes_regression'
 DATASET_NAME = 'diabetes_ds'
 DATASTORE_NAME = 'datablobstore'
 DATAFILE_NAME = 'diabetes.csv'

.pipelines/azdo-pr-build-train.yml

@@ -11,7 +11,7 @@ container: mcr.microsoft.com/mlops/python:latest
 
 
 variables:
-- template: azdo-variables.yml
+- template: diabetes_regression-variables.yml
 - group: devopsforai-aml-vg
 
 

.pipelines/azdo-ci-build-train.yml renamed to .pipelines/diabetes_regression-ci-build-train.yml

@@ -4,14 +4,14 @@ trigger:
     include:
     - master
   paths:
-    exclude:
-    - docs/
-    - environment_setup/
-    - ml_service/util/create_scoring_image.*
-    - ml_service/util/smoke_test_scoring_service.py
+    include:
+    - diabetes_regression/
+    - ml_service/pipelines/diabetes_regression_build_train_pipeline.py
+    - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py
+    - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py
 
 variables:
-- template: azdo-variables.yml
+- template: diabetes_regression-variables.yml
 - group: devopsforai-aml-vg
 
 

.pipelines/azdo-ci-image.yml renamed to .pipelines/diabetes_regression-ci-image.yml

@@ -7,10 +7,10 @@ trigger:
     include:
     - ml_service/util/create_scoring_image.py
     - ml_service/util/Dockerfile
-    - code/scoring/
+    - diabetes_regression/scoring/
     exclude:
-    - code/scoring/deployment_config_aci.yml
-    - code/scoring/deployment_config_aks.yml    
+    - diabetes_regression/scoring/deployment_config_aci.yml
+    - diabetes_regression/scoring/deployment_config_aks.yml    
 
 pool: 
   vmImage: 'ubuntu-latest'

.pipelines/azdo-variables.yml renamed to .pipelines/diabetes_regression-variables.yml

@@ -15,7 +15,7 @@ variables:
   value: lowpriority
   # Training Config
 - name: BUILD_TRAIN_SCRIPT
-  value: build_train_pipeline.py
+  value: diabetes_regression_build_train_pipeline.py
 - name: TRAIN_SCRIPT_PATH
   value: training/train.py
 - name: MODEL_NAME
@@ -24,17 +24,17 @@ variables:
   value: '1'
   # AML Pipeline Config 
 - name: TRAINING_PIPELINE_NAME
-  value: 'Training-Pipeline'
+  value: 'diabetes-Training-Pipeline'
 - name: MODEL_PATH
   value: ''
 - name: EVALUATE_SCRIPT_PATH
   value: evaluate/evaluate_model.py
 - name: REGISTER_SCRIPT_PATH
   value: register/register_model.py
 - name: SOURCES_DIR_TRAIN
-  value: code
+  value: diabetes_regression
 - name: IMAGE_NAME
-  value: 'mltrained'
+  value: 'diabetestrained'
   # Optional. Used by a training pipeline with R on Databricks
 - name: DB_CLUSTER_ID
   value: ''

code/evaluate/evaluate_model.py renamed to diabetes_regression/evaluate/evaluate_model.py

@@ -36,7 +36,7 @@
     load_dotenv()
     sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
     if (sources_dir is None):
-        sources_dir = 'code'
+        sources_dir = 'diabetes_regression'
     path_to_util = os.path.join(".", sources_dir, "util")
     sys.path.append(os.path.abspath(path_to_util))  # NOQA: E402
     from model_helper import get_model_by_tag