Skip to content

Commit 2a22e75

Browse files
algattikdtzar
algattik
authored and
dtzar
committed
Multi-stage pipeline + scoring observability (#122)
1 parent 11716e8 commit 2a22e75

14 files changed

+527
-346
lines changed

‎.pipelines/azdo-ci-build-train.yml

Lines changed: 107 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ trigger:
88
- docs/
99
- environment_setup/
1010
- ml_service/util/create_scoring_image.*
11+
- ml_service/util/smoke_test_scoring_service.py
1112

1213
variables:
1314
- template: azdo-variables.yml
@@ -36,8 +37,9 @@ stages:
3637
# Invoke the Python building and publishing a training pipeline
3738
python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }}
3839
displayName: 'Publish Azure Machine Learning Pipeline'
40+
3941
- stage: 'Trigger_AML_Pipeline'
40-
displayName: 'Train, evaluate, register model via previously published AML pipeline'
42+
displayName: 'Train model'
4143
jobs:
4244
- job: "Get_Pipeline_ID"
4345
condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true'))
@@ -87,32 +89,121 @@ stages:
8789
- job: "Training_Run_Report"
8890
dependsOn: "Run_ML_Pipeline"
8991
displayName: "Determine if evaluation succeeded and new model is registered"
92+
pool:
93+
vmImage: 'ubuntu-latest'
94+
container: mcr.microsoft.com/mlops/python:latest
95+
timeoutInMinutes: 0
96+
steps:
97+
- template: azdo-template-get-model-version.yml
98+
- stage: 'Deploy_ACI'
99+
displayName: 'Deploy to ACI'
100+
dependsOn: Trigger_AML_Pipeline
101+
condition: and(succeeded(), variables['ACI_DEPLOYMENT_NAME'])
102+
jobs:
103+
- job: "Deploy_ACI"
104+
displayName: "Deploy to ACI"
105+
pool:
106+
vmImage: 'ubuntu-latest'
107+
container: mcr.microsoft.com/mlops/python:latest
108+
timeoutInMinutes: 0
109+
steps:
110+
- template: azdo-template-get-model-version.yml
111+
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
112+
displayName: 'Azure ML Model Deploy'
113+
inputs:
114+
azureSubscription: $(WORKSPACE_SVC_CONNECTION)
115+
modelSourceType: manualSpec
116+
modelName: '$(MODEL_NAME)'
117+
modelVersion: $(MODEL_VERSION)
118+
inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
119+
deploymentTarget: ACI
120+
deploymentName: $(ACI_DEPLOYMENT_NAME)
121+
deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aci.yml'
122+
overwriteExistingDeployment: true
123+
- task: AzureCLI@1
124+
displayName: 'Smoke test'
125+
inputs:
126+
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
127+
scriptLocation: inlineScript
128+
inlineScript: |
129+
set -e # fail on error
130+
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
131+
python ml_service/util/smoke_test_scoring_service.py --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
132+
133+
- stage: 'Deploy_AKS'
134+
displayName: 'Deploy to AKS'
135+
dependsOn: Deploy_ACI
136+
condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
137+
jobs:
138+
- job: "Deploy_AKS"
139+
displayName: "Deploy to AKS"
90140
pool:
91141
vmImage: 'ubuntu-latest'
92142
container: mcr.microsoft.com/mlops/python:latest
93143
timeoutInMinutes: 0
94144
steps:
145+
- template: azdo-template-get-model-version.yml
146+
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
147+
displayName: 'Azure ML Model Deploy'
148+
inputs:
149+
azureSubscription: $(WORKSPACE_SVC_CONNECTION)
150+
modelSourceType: manualSpec
151+
modelName: '$(MODEL_NAME)'
152+
modelVersion: $(MODEL_VERSION)
153+
inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
154+
deploymentTarget: AKS
155+
aksCluster: $(AKS_COMPUTE_NAME)
156+
deploymentName: $(AKS_DEPLOYMENT_NAME)
157+
deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aks.yml'
158+
overwriteExistingDeployment: true
95159
- task: AzureCLI@1
160+
displayName: 'Smoke test'
96161
inputs:
97162
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
98163
scriptLocation: inlineScript
99164
inlineScript: |
100165
set -e # fail on error
101166
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
102-
python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId)
103-
displayName: "Determine if evaluation succeeded and new model is registered"
104-
- task: CopyFiles@2
105-
displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
167+
python ml_service/util/smoke_test_scoring_service.py --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
168+
169+
- stage: 'Deploy_Webapp'
170+
displayName: 'Deploy to Webapp'
171+
dependsOn: Trigger_AML_Pipeline
172+
condition: and(succeeded(), variables['WEBAPP_DEPLOYMENT_NAME'])
173+
jobs:
174+
- job: "Deploy_Webapp"
175+
displayName: "Deploy to Webapp"
176+
pool:
177+
vmImage: 'ubuntu-latest'
178+
container: mcr.microsoft.com/mlops/python:latest
179+
timeoutInMinutes: 0
180+
steps:
181+
- template: azdo-template-get-model-version.yml
182+
- task: AzureCLI@1
183+
displayName: 'Create scoring image and set IMAGE_LOCATION variable'
106184
inputs:
107-
SourceFolder: '$(Build.SourcesDirectory)'
108-
TargetFolder: '$(Build.ArtifactStagingDirectory)'
109-
Contents: |
110-
code/scoring/**
111-
ml_service/util/**
112-
- task: PublishBuildArtifacts@1
113-
displayName: 'Publish Artifact'
185+
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
186+
scriptLocation: inlineScript
187+
inlineScript: |
188+
set -e # fail on error
189+
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
190+
python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt
191+
# Output image location to Azure DevOps job
192+
IMAGE_LOCATION="$(cat image_location.txt)"
193+
echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
194+
- task: AzureWebAppContainer@1
195+
name: WebAppDeploy
196+
displayName: 'Azure Web App on Container Deploy'
197+
inputs:
198+
azureSubscription: 'AzureResourceConnection'
199+
appName: '$(WEBAPP_DEPLOYMENT_NAME)'
200+
containers: '$(IMAGE_LOCATION)'
201+
- task: AzureCLI@1
202+
displayName: 'Smoke test'
114203
inputs:
115-
ArtifactName: 'mlops-pipelines'
116-
publishLocation: 'container'
117-
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
118-
TargetPath: '$(Build.ArtifactStagingDirectory)'
204+
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
205+
scriptLocation: inlineScript
206+
inlineScript: |
207+
set -e # fail on error
208+
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
209+
python ml_service/util/smoke_test_scoring_service.py --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"

‎.pipelines/azdo-template-get-model-version.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
steps:
2+
- task: AzureCLI@1
3+
inputs:
4+
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
5+
scriptLocation: inlineScript
6+
inlineScript: |
7+
set -e # fail on error
8+
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
9+
python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt"
10+
# Output model version to Azure DevOps job
11+
MODEL_VERSION="$(cat model_version.txt)"
12+
echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION"
13+
name: 'getversion'
14+
displayName: "Determine if evaluation succeeded and new model is registered"

‎code/scoring/score.py

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,33 @@ def init():
3838
model = joblib.load(model_path)
3939

4040

41-
def run(raw_data):
41+
def run(raw_data, request_headers):
4242
data = json.loads(raw_data)["data"]
4343
data = numpy.array(data)
4444
result = model.predict(data)
45+
46+
# Demonstrate how we can log custom data into the Application Insights
47+
# traces collection.
48+
# The 'X-Ms-Request-id' value is generated internally and can be used to
49+
# correlate a log entry with the Application Insights requests collection.
50+
# The HTTP 'traceparent' header may be set by the caller to implement
51+
# distributed tracing (per the W3C Trace Context proposed specification)
52+
# and can be used to correlate the request to external systems.
53+
print(('{{"RequestId":"{0}", '
54+
'"TraceParent":"{1}", '
55+
'"NumberOfPredictions":{2}}}'
56+
).format(
57+
request_headers.get("X-Ms-Request-Id", ""),
58+
request_headers.get("Traceparent", ""),
59+
len(result)
60+
))
61+
4562
return {"result": result.tolist()}
4663

4764

4865
if __name__ == "__main__":
4966
# Test scoring
5067
init()
5168
test_row = '{"data":[[1,2,3,4,5,6,7,8,9,10],[10,9,8,7,6,5,4,3,2,1]]}'
52-
prediction = run(test_row)
69+
prediction = run(test_row, {})
5370
print("Test result: ", prediction)

0 commit comments

Comments
 (0)