Merge pull request #2 from Microsoft/buildpipeline

Buildpipeline

Author: praneet22

README.md

@@ -2,6 +2,8 @@
 
 # DevOps For AI
 
+[![Build Status](https://teamdanielletrialcicd.visualstudio.com/MLAKSDeployAML/_apis/build/status/DevopsforAI-AML?branchName=master)](https://teamdanielletrialcicd.visualstudio.com/MLAKSDeployAML/_build/latest?definitionId=9&branchName=buildpipeline)
+
 [DevOps for AI template](https://azuredevopsdemogenerator.azurewebsites.net/?name=azure%20machine%20learning) will help you to understand how to build the Continuous Integration and Continuous Delivery pipeline for a ML/AI project. We will be using the Azure DevOps Project for build and release pipelines along with Azure ML services for ML/AI model management and operationalization. 
 
 This template contains code and pipeline definition for a machine learning project demonstrating how to automate the end to end ML/AI project. The build pipelines include DevOps tasks for data sanity test, unit test, model training on different compute targets, model version management, model evaluation/model selection, model deployment as realtime web service, staged deployment to QA/prod, integration testing and functional testing.

aml_service/00-WorkSpace.py

@@ -24,8 +24,9 @@
 POSSIBILITY OF SUCH DAMAGE.
 """
 from azureml.core import Workspace
-import os, json
+import os, json, sys
 import azureml.core
+from azureml.core.authentication import AzureCliAuthentication
 
 print("SDK Version:", azureml.core.VERSION)
 # print('current dir is ' +os.curdir)
@@ -36,12 +37,15 @@
 resource_group = config["resource_group"]
 subscription_id = config["subscription_id"]
 location = config["location"]
-# location = 'southcentralus'
+
+cli_auth = AzureCliAuthentication()
+
 try:
     ws = Workspace.get(
         name=workspace_name,
         subscription_id=subscription_id,
         resource_group=resource_group,
+        auth=cli_auth
     )
 
 except:
@@ -53,6 +57,8 @@
         resource_group=resource_group,
         # create_resource_group=True,
         location=location,
+        auth=cli_auth
+
     )
 
 # print Workspace details

aml_service/10-TrainOnLocal.py

@@ -29,9 +29,11 @@
 from azureml.core import Experiment
 from azureml.core import ScriptRunConfig
 import json
+from azureml.core.authentication import AzureCliAuthentication
+cli_auth = AzureCliAuthentication()
 
 # Get workspace
-ws = Workspace.from_config()
+ws = Workspace.from_config(auth=cli_auth)
 
 # Attach Experiment
 experiment_name = "devops-ai-demo"

aml_service/15-EvaluateModel.py

@@ -29,10 +29,11 @@
 from azureml.core.model import Model
 import azureml.core
 from azureml.core import Run
-
+from azureml.core.authentication import AzureCliAuthentication
+cli_auth = AzureCliAuthentication()
 
 # Get workspace
-ws = Workspace.from_config()
+ws = Workspace.from_config(auth=cli_auth)
 
 # Paramaterize the matrics on which the models should be compared
 

aml_service/20-RegisterModel.py

@@ -30,9 +30,11 @@
 from azureml.core.model import Model
 
 from azureml.core.runconfig import RunConfiguration
+from azureml.core.authentication import AzureCliAuthentication
+cli_auth = AzureCliAuthentication()
 
 # Get workspace
-ws = Workspace.from_config()
+ws = Workspace.from_config(auth=cli_auth)
 
 # Get the latest evaluation result
 try:

aml_service/30-CreateScoringImage.py

@@ -27,9 +27,11 @@
 from azureml.core import Workspace
 from azureml.core.image import ContainerImage, Image
 from azureml.core.model import Model
+from azureml.core.authentication import AzureCliAuthentication
+cli_auth = AzureCliAuthentication()
 
 # Get workspace
-ws = Workspace.from_config()
+ws = Workspace.from_config(auth=cli_auth)
 
 # Get the latest model details
 

azure-pipelines.yml

@@ -0,0 +1,68 @@
+pool:
+  vmImage: 'Ubuntu 16.04'
+#Your build pipeline references a secret variable named ‘sp_username’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
+#Your build pipeline references a secret variable named ‘sp_password’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
+#Your build pipeline references a secret variable named ‘sp_tenantid’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
+#Your build pipeline references a secret variable named ‘subscription_id’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
+
+steps:
+- task: CondaEnvironment@1
+  displayName: 'Create Conda Environment '
+  inputs:
+    createCustomEnvironment: true
+    environmentName: azuremlsdk
+    packageSpecs: 'python=3.6'
+    updateConda: false
+    createOptions: 'cython numpy'
+
+- task: Bash@3
+  displayName: 'Install Requirements'
+  inputs:
+    targetType: filePath
+    filePath: 'environment_setup/install_requirements.sh'
+    workingDirectory: 'environment_setup'
+
+- script: |
+   az login --service-principal -u $(sp_username) -p $(sp_password) --tenant $(sp_tenantid)
+   
+  displayName: 'Login to Azure'
+
+- script: |
+   sed -i 's#"subscription_id": "<>"#"subscription_id": "$(subscription_id)"#g' aml_config/config.json
+   
+  displayName: 'replace subscription value'
+
+- script: 'python code/testing/data_test.py data/diabetes.csv && python code/testing/data_test.py data/diabetes_bad_dist.csv && python code/testing/data_test.py data/diabetes_bad_schema.csv && python code/testing/data_test.py data/diabetes_missing_values.csv'
+  displayName: 'Data Quality Check'
+
+- script: 'python aml_service/00-WorkSpace.py'
+  displayName: 'Get or Create workspace copy'
+
+- script: 'python aml_service/10-TrainOnLocal.py'
+  displayName: 'Train on Local'
+
+- script: 'python aml_service/15-EvaluateModel.py'
+  displayName: 'Evaluate Model'
+
+- script: 'python aml_service/20-RegisterModel.py'
+  displayName: 'Register Model'
+
+- script: 'python aml_service/30-CreateScoringImage.py'
+  displayName: 'Creating scoring image'
+
+- task: CopyFiles@2
+  displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)'
+    TargetFolder: '$(Build.ArtifactStagingDirectory)'
+    Contents: '**'
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: devops-for-ai'
+  inputs:
+    ArtifactName: 'devops-for-ai'
+    publishLocation: 'container'
+    pathtoPublish: '$(Build.ArtifactStagingDirectory)' 
+    TargetPath: '$(Build.ArtifactStagingDirectory)'
+
+