extending use of SOURCES_DIR_TRAIN variable (#143)

Author: jotaylo

.pipelines/azdo-ci-build-train.yml

@@ -115,10 +115,10 @@ stages:
         modelSourceType: manualSpec
         modelName: '$(MODEL_NAME)'
         modelVersion: $(MODEL_VERSION)
-        inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
+        inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
         deploymentTarget: ACI
         deploymentName: $(ACI_DEPLOYMENT_NAME)
-        deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aci.yml'
+        deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aci.yml'
         overwriteExistingDeployment: true
     - task: AzureCLI@1
       displayName: 'Smoke test'
@@ -150,11 +150,11 @@ stages:
         modelSourceType: manualSpec
         modelName: '$(MODEL_NAME)'
         modelVersion: $(MODEL_VERSION)
-        inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
+        inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
         deploymentTarget: AKS
         aksCluster: $(AKS_COMPUTE_NAME)
         deploymentName: $(AKS_DEPLOYMENT_NAME)
-        deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aks.yml'
+        deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aks.yml'
         overwriteExistingDeployment: true
     - task: AzureCLI@1
       displayName: 'Smoke test'

code/evaluate/evaluate_model.py

@@ -32,10 +32,14 @@
 run = Run.get_context()
 if (run.id.startswith('OfflineRun')):
     from dotenv import load_dotenv
-    sys.path.append(os.path.abspath("./code/util"))  # NOQA: E402
-    from model_helper import get_model_by_tag
     # For local development, set values in this section
     load_dotenv()
+    sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
+    if (sources_dir is None):
+        sources_dir = 'code'
+    path_to_util = os.path.join(".", sources_dir, "util")
+    sys.path.append(os.path.abspath(path_to_util))  # NOQA: E402
+    from model_helper import get_model_by_tag
     workspace_name = os.environ.get("WORKSPACE_NAME")
     experiment_name = os.environ.get("EXPERIMENT_NAME")
     resource_group = os.environ.get("RESOURCE_GROUP")

ml_service/pipelines/verify_train_pipeline.py

@@ -11,10 +11,14 @@ def main():
     run = Run.get_context()
     if (run.id.startswith('OfflineRun')):
         from dotenv import load_dotenv
-        sys.path.append(os.path.abspath("./code/util"))  # NOQA: E402
-        from model_helper import get_model_by_tag
         # For local development, set values in this section
         load_dotenv()
+        sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
+        if (sources_dir is None):
+            sources_dir = 'code'
+        path_to_util = os.path.join(".", sources_dir, "util")
+        sys.path.append(os.path.abspath(path_to_util))  # NOQA: E402
+        from model_helper import get_model_by_tag
         workspace_name = os.environ.get("WORKSPACE_NAME")
         experiment_name = os.environ.get("EXPERIMENT_NAME")
         resource_group = os.environ.get("RESOURCE_GROUP")

ml_service/util/create_scoring_image.py

@@ -26,7 +26,12 @@
 args = parser.parse_args()
 
 model = Model(ws, name=e.model_name, version=e.model_version)
-os.chdir("./code/scoring")
+sources_dir = e.sources_dir
+if (sources_dir is None):
+    sources_dir = 'code'
+path_to_scoring = os.path.join(".", sources_dir, "scoring")
+cwd = os.getcwd()
+os.chdir(path_to_scoring)
 
 image_config = ContainerImage.image_configuration(
     execution_script=e.score_script,
@@ -40,7 +45,7 @@
     name=e.image_name, models=[model], image_config=image_config, workspace=ws
 )
 
-os.chdir("../..")
+os.chdir(cwd)
 
 image.wait_for_creation(show_output=True)