Skip to content

Commit 421ef73

Browse files
praneet22praneet22
authored
Merge pull request #7 from Microsoft/feature/aml-pipeline
Feature/aml pipeline
2 parents 17bc04e + c439c60 commit 421ef73

File tree

12 files changed

+798
-35
lines changed

12 files changed

+798
-35
lines changed

‎aml_config/security_config.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
"remote_vm_username" : "<>",
77
"remote_vm_password" : "<>",
88
"remote_vm_ip" : "<>",
9-
"experiment_name" : "<>",
10-
"aml_cluster_name" : "<>",
9+
"experiment_name" : "devops-ai-demo",
10+
"aml_cluster_name" : "aml-compute",
1111
"vnet_resourcegroup_name" : "<>",
1212
"vnet_name" : "<>",
1313
"subnet_name" : "<>"

‎aml_service/04-AmlPipelines.py‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
"""
2+
Copyright (C) Microsoft Corporation. All rights reserved.​
3+
​
4+
Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual,
5+
royalty-free right to use, copy, and modify the software code provided by us
6+
("Software Code"). You may not sublicense the Software Code or any use of it
7+
(except to your affiliates and to vendors to perform work on your behalf)
8+
through distribution, network access, service agreement, lease, rental, or
9+
otherwise. This license does not purport to express any claim of ownership over
10+
data you may have shared with Microsoft in the creation of the Software Code.
11+
Unless applicable law gives you more rights, Microsoft reserves all other
12+
rights not expressly granted herein, whether by implication, estoppel or
13+
otherwise. ​
14+
​
15+
THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
16+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18+
MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
21+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22+
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23+
ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
24+
POSSIBILITY OF SUCH DAMAGE.
25+
"""
26+
27+
import os, json, requests, datetime
28+
import argparse
29+
from azureml.core import Workspace, Experiment, Datastore
30+
from azureml.core.runconfig import RunConfiguration, CondaDependencies
31+
from azureml.data.data_reference import DataReference
32+
from azureml.pipeline.core import Pipeline, PipelineData, StepSequence
33+
from azureml.pipeline.steps import PythonScriptStep
34+
from azureml.pipeline.core import PublishedPipeline
35+
from azureml.pipeline.core.graph import PipelineParameter
36+
from azureml.core.compute import ComputeTarget
37+
38+
# from azureml.widgets import RunDetails
39+
from azureml.core.authentication import AzureCliAuthentication
40+
41+
print("Pipeline SDK-specific imports completed")
42+
43+
cli_auth = AzureCliAuthentication()
44+
45+
46+
parser = argparse.ArgumentParser("Pipeline")
47+
parser.add_argument(
48+
"--pipeline_action",
49+
type=str,
50+
choices=["pipeline-test", "publish"],
51+
help="Determines if pipeline needs to run on small data set \
52+
or pipeline needs to be republished",
53+
#default="pipeline-test",
54+
)
55+
56+
args = parser.parse_args()
57+
58+
59+
# Get workspace
60+
ws = Workspace.from_config(path="aml_config/config.json", auth=cli_auth)
61+
def_blob_store = Datastore(ws, "workspaceblobstore")
62+
63+
# Get AML Compute name and Experiment Name
64+
with open("aml_config/security_config.json") as f:
65+
config = json.load(f)
66+
67+
experiment_name = config["experiment_name"]
68+
aml_cluster_name = config["aml_cluster_name"]
69+
aml_pipeline_name = "training-pipeline"
70+
71+
source_directory = "code"
72+
73+
# Run Config
74+
# Declare packages dependencies required in the pipeline (these can also be expressed as a YML file)
75+
# cd = CondaDependencies.create(pip_packages=["azureml-defaults", 'tensorflow==1.8.0'])
76+
cd = CondaDependencies("aml_config/conda_dependencies.yml")
77+
78+
run_config = RunConfiguration(conda_dependencies=cd)
79+
80+
aml_compute = ws.compute_targets[aml_cluster_name]
81+
82+
jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store)
83+
84+
# Suffix for all the config files
85+
config_suffix = datetime.datetime.now().strftime("%Y%m%d%H")
86+
print("PipelineData object created")
87+
88+
# Create python script step to run the training/scoring main script
89+
train = PythonScriptStep(
90+
name="Train New Model",
91+
script_name="training/train.py",
92+
compute_target=aml_compute,
93+
source_directory=source_directory,
94+
arguments=["--config_suffix", config_suffix, "--json_config", jsonconfigs],
95+
runconfig=run_config,
96+
# inputs=[jsonconfigs],
97+
outputs=[jsonconfigs],
98+
allow_reuse=False,
99+
)
100+
print("Step Train created")
101+
102+
evaluate = PythonScriptStep(
103+
name="Evaluate New Model with Prod Model",
104+
script_name="evaluate/evaluate_model.py",
105+
compute_target=aml_compute,
106+
source_directory=source_directory,
107+
arguments=["--config_suffix", config_suffix, "--json_config", jsonconfigs],
108+
runconfig=run_config,
109+
inputs=[jsonconfigs],
110+
# outputs=[jsonconfigs],
111+
allow_reuse=False,
112+
)
113+
print("Step Evaluate created")
114+
115+
register_model = PythonScriptStep(
116+
name="Register New Trained Model",
117+
script_name="register/register_model.py",
118+
compute_target=aml_compute,
119+
source_directory=source_directory,
120+
arguments=["--config_suffix", config_suffix, "--json_config", jsonconfigs],
121+
runconfig=run_config,
122+
inputs=[jsonconfigs],
123+
# outputs=[jsonconfigs],
124+
allow_reuse=False,
125+
)
126+
print("Step register model created")
127+
128+
package_model = PythonScriptStep(
129+
name="Package Model as Scoring Image",
130+
script_name="scoring/create_scoring_image.py",
131+
compute_target=aml_compute,
132+
source_directory=source_directory,
133+
arguments=["--config_suffix", config_suffix, "--json_config", jsonconfigs],
134+
runconfig=run_config,
135+
inputs=[jsonconfigs],
136+
# outputs=[jsonconfigs],
137+
allow_reuse=False,
138+
)
139+
print("Packed the model into a Scoring Image")
140+
141+
# Create Steps dependency such that they run in sequence
142+
evaluate.run_after(train)
143+
register_model.run_after(evaluate)
144+
package_model.run_after(register_model)
145+
146+
steps = [package_model]
147+
148+
149+
# Build Pipeline
150+
pipeline1 = Pipeline(workspace=ws, steps=steps)
151+
print("Pipeline is built")
152+
153+
# Validate Pipeline
154+
pipeline1.validate()
155+
print("Pipeline validation complete")
156+
157+
158+
# Submit unpublished pipeline with small data set for test
159+
if args.pipeline_action == "pipeline-test":
160+
pipeline_run1 = Experiment(ws, experiment_name).submit(
161+
pipeline1, regenerate_outputs=True
162+
)
163+
print("Pipeline is submitted for execution")
164+
pipeline_run1.wait_for_completion(show_output=True)
165+
166+
167+
# RunDetails(pipeline_run1).show()
168+
169+
170+
# Define pipeline parameters
171+
# run_env = PipelineParameter(
172+
# name="dev_flag",
173+
# default_value=True)
174+
175+
# dbname = PipelineParameter(
176+
# name="dbname",
177+
# default_value='opex')
178+
179+
180+
# Publish Pipeline
181+
if args.pipeline_action == "publish":
182+
published_pipeline1 = pipeline1.publish(
183+
name=aml_pipeline_name, description="Model training/retraining pipeline"
184+
)
185+
print(
186+
"Pipeline is published as rest_endpoint {} ".format(
187+
published_pipeline1.endpoint
188+
)
189+
)
190+
# write published pipeline details as build artifact
191+
pipeline_config = {}
192+
pipeline_config["pipeline_name"] = published_pipeline1.name
193+
pipeline_config["rest_endpoint"] = published_pipeline1.endpoint
194+
pipeline_config["experiment_name"] = "published-pipeline-exp" # experiment_name
195+
with open("aml_config/pipeline_config.json", "w") as outfile:
196+
json.dump(pipeline_config, outfile)

‎aml_service/05-TriggerAmlPipeline.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
"""
2+
Copyright (C) Microsoft Corporation. All rights reserved.​
3+
​
4+
Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual,
5+
royalty-free right to use, copy, and modify the software code provided by us
6+
("Software Code"). You may not sublicense the Software Code or any use of it
7+
(except to your affiliates and to vendors to perform work on your behalf)
8+
through distribution, network access, service agreement, lease, rental, or
9+
otherwise. This license does not purport to express any claim of ownership over
10+
data you may have shared with Microsoft in the creation of the Software Code.
11+
Unless applicable law gives you more rights, Microsoft reserves all other
12+
rights not expressly granted herein, whether by implication, estoppel or
13+
otherwise. ​
14+
​
15+
THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
16+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18+
MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
21+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22+
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23+
ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
24+
POSSIBILITY OF SUCH DAMAGE.
25+
"""
26+
27+
import os, json, requests, datetime, sys
28+
import argparse
29+
from azureml.core.authentication import AzureCliAuthentication
30+
31+
try:
32+
with open("aml_config/pipeline_config.json") as f:
33+
config = json.load(f)
34+
except:
35+
print("No pipeline config found")
36+
sys.exit(0)
37+
38+
# Run a published pipeline
39+
cli_auth = AzureCliAuthentication()
40+
aad_token = cli_auth.get_authentication_header()
41+
rest_endpoint1 = config["rest_endpoint"]
42+
experiment_name = config["experiment_name"]
43+
print(rest_endpoint1)
44+
45+
response = requests.post(
46+
rest_endpoint1, headers=aad_token, json={"ExperimentName": experiment_name}
47+
)
48+
49+
run_id = response.json()["Id"]
50+
print(run_id)
51+
print("Pipeline run initiated")

‎aml_service/30-CreateScoringImage.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from azureml.core.image import ContainerImage, Image
2929
from azureml.core.model import Model
3030
from azureml.core.authentication import AzureCliAuthentication
31+
3132
cli_auth = AzureCliAuthentication()
3233

3334
# Get workspace

‎azure-pipeline-yaml/retraining-trigger-pipelines.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
pool:
2+
vmImage: 'Ubuntu 16.04'
3+
#Your build pipeline references a secret variable named ‘sp_username’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
4+
#Your build pipeline references a secret variable named ‘sp_password’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
5+
#Your build pipeline references a secret variable named ‘sp_tenantid’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
6+
#Your build pipeline references a secret variable named ‘subscription_id’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
7+
8+
steps:
9+
- task: UsePythonVersion@0
10+
inputs:
11+
versionSpec: '3.6'
12+
architecture: 'x64'
13+
14+
# - task: DownloadPipelineArtifact@0
15+
# inputs:
16+
# artifactName: 'devops-for-ai-aml-pipeline'
17+
# targetPath: '$(System.DefaultWorkingDirectory)/aml_config/pipeline_config.json'
18+
19+
- task: Bash@3
20+
displayName: 'Install Requirements'
21+
inputs:
22+
targetType: filePath
23+
filePath: 'environment_setup/install_requirements.sh'
24+
workingDirectory: 'environment_setup'
25+
26+
- script: |
27+
az login --service-principal -u $(sp_username) -p $(sp_password) --tenant $(sp_tenantid)
28+
29+
displayName: 'Login to Azure'
30+
31+
- script: |
32+
sed -i 's#"subscription_id": "<>"#"subscription_id": "$(subscription_id)"#g' aml_config/config.json
33+
34+
displayName: 'replace subscription value'
35+
36+
- script: 'python aml_service/05-TriggerAmlPipeline.py'
37+
displayName: 'Trigger retraining Azure ML Pipeline'
38+
39+
- task: CopyFiles@2
40+
displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
41+
inputs:
42+
SourceFolder: '$(Build.SourcesDirectory)'
43+
TargetFolder: '$(Build.ArtifactStagingDirectory)'
44+
Contents: '**'
45+
46+
- task: PublishBuildArtifacts@1
47+
displayName: 'Publish Artifact: AML Pipeline Run'
48+
inputs:
49+
ArtifactName: 'aml-pipeline-run'
50+
publishLocation: 'container'
51+
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
52+
TargetPath: '$(Build.ArtifactStagingDirectory)'
53+
54+

‎azure-pipeline-yaml/simple-azure-pipelines.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
pool:
2+
vmImage: 'Ubuntu 16.04'
3+
#Your build pipeline references a secret variable named ‘sp_username’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
4+
#Your build pipeline references a secret variable named ‘sp_password’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
5+
#Your build pipeline references a secret variable named ‘sp_tenantid’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
6+
#Your build pipeline references a secret variable named ‘subscription_id’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972
7+
8+
steps:
9+
- task: UsePythonVersion@0
10+
inputs:
11+
versionSpec: '3.6'
12+
architecture: 'x64'
13+
14+
- task: Bash@3
15+
displayName: 'Install Requirements'
16+
inputs:
17+
targetType: filePath
18+
filePath: 'environment_setup/install_requirements.sh'
19+
workingDirectory: 'environment_setup'
20+
21+
- script: |
22+
az login --service-principal -u $(sp_username) -p $(sp_password) --tenant $(sp_tenantid)
23+
24+
displayName: 'Login to Azure'
25+
26+
- script: |
27+
sed -i 's#"subscription_id": "<>"#"subscription_id": "$(subscription_id)"#g' aml_config/config.json
28+
29+
displayName: 'replace subscription value'
30+
31+
- script: 'pytest tests/unit/data_test.py'
32+
displayName: 'Data Quality Check'
33+
34+
- script: 'python aml_service/00-WorkSpace.py'
35+
displayName: 'Get or Create workspace copy'
36+
37+
- script: 'python aml_service/10-TrainOnLocal.py'
38+
displayName: 'Train on Local'
39+
40+
- script: 'python aml_service/15-EvaluateModel.py'
41+
displayName: 'Evaluate Model'
42+
43+
- script: 'python aml_service/20-RegisterModel.py'
44+
displayName: 'Register Model'
45+
46+
- script: 'python aml_service/30-CreateScoringImage.py'
47+
displayName: 'Creating scoring image'
48+
49+
- task: CopyFiles@2
50+
displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
51+
inputs:
52+
SourceFolder: '$(Build.SourcesDirectory)'
53+
TargetFolder: '$(Build.ArtifactStagingDirectory)'
54+
Contents: '**'
55+
56+
- task: PublishBuildArtifacts@1
57+
displayName: 'Publish Artifact: devops-for-ai'
58+
inputs:
59+
ArtifactName: 'devops-for-ai'
60+
publishLocation: 'container'
61+
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
62+
TargetPath: '$(Build.ArtifactStagingDirectory)'
63+
64+

0 commit comments

Comments
 (0)