feature: Updated AML Pipeline code and added AML Pipeline trigger step

Author: praneet22

aml_service/04-AmlPipelines.py

@@ -47,9 +47,10 @@
 parser.add_argument(
     "--pipeline_action",
     type=str,
-    choices=["unpublish-test", "publish"],
+    choices=["pipeline-test", "publish"],
     help="Determines if pipeline needs to run on small data set \
                                         or pipeline needs to be republished",
+    default="pipeline-test",
 )
 
 args = parser.parse_args()
@@ -140,9 +141,9 @@
 # Create Steps dependency such that they run in sequence
 evaluate.run_after(train)
 register_model.run_after(evaluate)
-create_scoring_image.run_after(register_model)
+package_model.run_after(register_model)
 
-steps = [create_scoring_image]
+steps = [package_model]
 
 
 # Build Pipeline
@@ -154,7 +155,7 @@
 print("Pipeline validation complete")
 
 # Submit unpublished pipeline with small data set for test
-if args.pipeline_action == "unpublish-test":
+if args.pipeline_action == "pipeline-test":
     pipeline_run1 = Experiment(ws, experiment_name).submit(
         pipeline1, regenerate_outputs=True
     )
@@ -164,7 +165,6 @@
 
 # RunDetails(pipeline_run1).show()
 
-## Publish Pipeline
 
 # Define pipeline parameters
 # run_env = PipelineParameter(
@@ -177,22 +177,19 @@
 
 
 # Publish Pipeline
-
-published_pipeline1 = pipeline1.publish(
-    name=aml_pipeline_name, description="Model training/retraining pipeline"
-)
-
-
-# # Run a piblished pipeline
-# cli_auth = AzureCliAuthentication()
-# aad_token = cli_auth.get_authentication_header()
-# rest_endpoint1 = published_pipeline1.endpoint
-# print(rest_endpoint1)
-
-# response = requests.post(published_pipeline1.endpoint,
-#     headers=aad_token,
-#     json={"ExperimentName": "devops-ai",
-#         "ParameterAssignments": {"dev_flag": True}})
-
-# run_id = response.json()["Id"]
-# print(run_id)
+if args.pipeline_action == "publish":
+    published_pipeline1 = pipeline1.publish(
+        name=aml_pipeline_name, description="Model training/retraining pipeline"
+    )
+    print(
+        "Pipeline is published as rest_endpoint {} ".format(
+            published_pipeline1.endpoint
+        )
+    )
+    # write published pipeline details as build artifact
+    pipeline_config = {}
+    pipeline_config["pipeline_name"] = published_pipeline1.name
+    pipeline_config["rest_endpoint"] = published_pipeline1.endpoint
+    pipeline_config["experiment_name"] = "published-pipeline-exp"  # experiment_name
+    with open("aml_config/pipeline_config.json", "w") as outfile:
+        json.dump(pipeline_config, outfile)

aml_service/05-TriggerAmlPipeline.py

@@ -0,0 +1,51 @@
+"""
+Copyright (C) Microsoft Corporation. All rights reserved.​
+ ​
+Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual,
+royalty-free right to use, copy, and modify the software code provided by us
+("Software Code"). You may not sublicense the Software Code or any use of it
+(except to your affiliates and to vendors to perform work on your behalf)
+through distribution, network access, service agreement, lease, rental, or
+otherwise. This license does not purport to express any claim of ownership over
+data you may have shared with Microsoft in the creation of the Software Code.
+Unless applicable law gives you more rights, Microsoft reserves all other
+rights not expressly granted herein, whether by implication, estoppel or
+otherwise. ​
+ ​
+THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+"""
+
+import os, json, requests, datetime
+import argparse
+from azureml.core.authentication import AzureCliAuthentication
+
+try:
+    with open("aml_config/pipeline_config.json") as f:
+        config = json.load(f)
+except:
+    print("No pipeline config found")
+    sys.exit(0)
+
+# Run a published pipeline
+cli_auth = AzureCliAuthentication()
+aad_token = cli_auth.get_authentication_header()
+rest_endpoint1 = config["rest_endpoint"]
+experiment_name = config["experiment_name"]
+print(rest_endpoint1)
+
+response = requests.post(
+    rest_endpoint1, headers=aad_token, json={"ExperimentName": experiment_name}
+)
+
+run_id = response.json()["Id"]
+print(run_id)
+print("Pipeline run initiated")

aml_service/30-CreateScoringImage.py

@@ -28,6 +28,7 @@
 from azureml.core.image import ContainerImage, Image
 from azureml.core.model import Model
 from azureml.core.authentication import AzureCliAuthentication
+
 cli_auth = AzureCliAuthentication()
 
 # Get workspace

code/scoring/create_scoring_image.py

@@ -78,7 +78,7 @@
     )
 )
 
-os.chdir("./code/scoring")
+os.chdir("scoring")
 image_name = "diabetes-model-score"
 
 image_config = ContainerImage.image_configuration(
@@ -94,7 +94,7 @@
 )
 
 image.wait_for_creation(show_output=True)
-os.chdir("../..")
+os.chdir("..")
 
 if image.creation_state != "Succeeded":
     raise Exception("Image creation status: {image.creation_state}")