Skip to content

Commit cfd3e2e

Browse files
praneet22praneet22
committed
feat: added aml pipeline steps script
1 parent a399664 commit cfd3e2e

File tree

4 files changed

+400
-9
lines changed

4 files changed

+400
-9
lines changed

‎code/evaluate/evaluate_model.py‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
"""
2+
Copyright (C) Microsoft Corporation. All rights reserved.​
3+
​
4+
Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual,
5+
royalty-free right to use, copy, and modify the software code provided by us
6+
("Software Code"). You may not sublicense the Software Code or any use of it
7+
(except to your affiliates and to vendors to perform work on your behalf)
8+
through distribution, network access, service agreement, lease, rental, or
9+
otherwise. This license does not purport to express any claim of ownership over
10+
data you may have shared with Microsoft in the creation of the Software Code.
11+
Unless applicable law gives you more rights, Microsoft reserves all other
12+
rights not expressly granted herein, whether by implication, estoppel or
13+
otherwise. ​
14+
​
15+
THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
16+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18+
MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
21+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22+
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23+
ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
24+
POSSIBILITY OF SUCH DAMAGE.
25+
"""
26+
import os, json
27+
from azureml.core import Workspace
28+
from azureml.core import Experiment
29+
from azureml.core.model import Model
30+
import azureml.core
31+
from azureml.core import Run
32+
import argparse
33+
34+
35+
# Get workspace
36+
# ws = Workspace.from_config()
37+
run = Run.get_context()
38+
exp = run.experiment
39+
ws = run.experiment.workspace
40+
41+
42+
parser = argparse.ArgumentParser("evaluate")
43+
parser.add_argument(
44+
"--config_suffix", type=str, help="Datetime suffix for json config files"
45+
)
46+
parser.add_argument(
47+
"--json_config",
48+
type=str,
49+
help="Directory to write all the intermediate json configs",
50+
)
51+
args = parser.parse_args()
52+
53+
print("Argument 1: %s" % args.config_suffix)
54+
print("Argument 2: %s" % args.json_config)
55+
56+
if not (args.json_config is None):
57+
os.makedirs(args.json_config, exist_ok=True)
58+
print("%s created" % args.json_config)
59+
# Paramaterize the matrics on which the models should be compared
60+
# Add golden data set on which all the model performance can be evaluated
61+
62+
# Get the latest run_id
63+
# with open("aml_config/run_id.json") as f:
64+
# config = json.load(f)
65+
66+
train_run_id_json = "run_id_{}.json".format(args.config_suffix)
67+
train_output_path = os.path.join(args.json_config, train_run_id_json)
68+
with open(train_output_path) as f:
69+
config = json.load(f)
70+
71+
# parser = argparse.ArgumentParser()
72+
# parser.add_argument('--train_run_id',type=str,default='',help='Run id of the newly trained model')
73+
# #parser.add_argument('--model_assets_path',type=str,default='outputs',help='Location of trained model.')
74+
75+
76+
new_model_run_id = config["run_id"] # args.train_run_id
77+
experiment_name = config["experiment_name"]
78+
# exp = Experiment(workspace=ws, name=experiment_name)
79+
80+
81+
try:
82+
# Get most recently registered model, we assume that is the model in production. Download this model and compare it with the recently trained model by running test with same data set.
83+
model_list = Model.list(ws)
84+
production_model = next(
85+
filter(
86+
lambda x: x.created_time == max(model.created_time for model in model_list),
87+
model_list,
88+
)
89+
)
90+
production_model_run_id = production_model.tags.get("run_id")
91+
run_list = exp.get_runs()
92+
# production_model_run = next(filter(lambda x: x.id == production_model_run_id, run_list))
93+
94+
# Get the run history for both production model and newly trained model and compare mse
95+
production_model_run = Run(exp, run_id=production_model_run_id)
96+
new_model_run = Run(exp, run_id=new_model_run_id)
97+
98+
production_model_mse = production_model_run.get_metrics().get("mse")
99+
new_model_mse = new_model_run.get_metrics().get("mse")
100+
print(
101+
"Current Production model mse: {}, New trained model mse: {}".format(
102+
production_model_mse, new_model_mse
103+
)
104+
)
105+
106+
promote_new_model = False
107+
if new_model_mse < production_model_mse:
108+
promote_new_model = True
109+
print("New trained model performs better, thus it will be registered")
110+
except:
111+
promote_new_model = True
112+
print("This is the first model to be trained, thus nothing to evaluate for now")
113+
114+
run_id = {}
115+
run_id["run_id"] = ""
116+
# Writing the run id to /aml_config/run_id.json
117+
if promote_new_model:
118+
run_id["run_id"] = new_model_run_id
119+
# register new model
120+
# new_model_run.register_model(model_name='',model_path='outputs/sklearn_regression_model.pkl')
121+
122+
run_id["experiment_name"] = experiment_name
123+
filename = "run_id_{}.json".format(args.config_suffix)
124+
output_path = os.path.join(args.json_config, filename)
125+
with open(output_path, "w") as outfile:
126+
json.dump(run_id, outfile)

‎code/register/register_model.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
"""
2+
Copyright (C) Microsoft Corporation. All rights reserved.​
3+
​
4+
Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual,
5+
royalty-free right to use, copy, and modify the software code provided by us
6+
("Software Code"). You may not sublicense the Software Code or any use of it
7+
(except to your affiliates and to vendors to perform work on your behalf)
8+
through distribution, network access, service agreement, lease, rental, or
9+
otherwise. This license does not purport to express any claim of ownership over
10+
data you may have shared with Microsoft in the creation of the Software Code.
11+
Unless applicable law gives you more rights, Microsoft reserves all other
12+
rights not expressly granted herein, whether by implication, estoppel or
13+
otherwise. ​
14+
​
15+
THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
16+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18+
MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
21+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22+
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23+
ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
24+
POSSIBILITY OF SUCH DAMAGE.
25+
"""
26+
import os, json, sys
27+
from azureml.core import Workspace
28+
from azureml.core import Run
29+
from azureml.core import Experiment
30+
from azureml.core.model import Model
31+
import argparse
32+
33+
from azureml.core.runconfig import RunConfiguration
34+
from azureml.core.authentication import AzureCliAuthentication
35+
36+
cli_auth = AzureCliAuthentication()
37+
38+
# Get workspace
39+
# ws = Workspace.from_config(auth=cli_auth)
40+
run = Run.get_context()
41+
exp = run.experiment
42+
ws = run.experiment.workspace
43+
44+
parser = argparse.ArgumentParser("register")
45+
parser.add_argument(
46+
"--config_suffix", type=str, help="Datetime suffix for json config files"
47+
)
48+
parser.add_argument(
49+
"--json_config",
50+
type=str,
51+
help="Directory to write all the intermediate json configs",
52+
)
53+
args = parser.parse_args()
54+
55+
print("Argument 1: %s" % args.config_suffix)
56+
print("Argument 2: %s" % args.json_config)
57+
58+
if not (args.json_config is None):
59+
os.makedirs(args.json_config, exist_ok=True)
60+
print("%s created" % args.json_config)
61+
62+
evaluate_run_id_json = "run_id_{}.json".format(args.config_suffix)
63+
evaluate_output_path = os.path.join(args.json_config, evaluate_run_id_json)
64+
65+
# Get the latest evaluation result
66+
try:
67+
with open(evaluate_output_path) as f:
68+
config = json.load(f)
69+
if not config["run_id"]:
70+
raise Exception("No new model to register as production model perform better")
71+
except:
72+
print("No new model to register as production model perform better")
73+
# raise Exception('No new model to register as production model perform better')
74+
sys.exit(0)
75+
76+
run_id = config["run_id"]
77+
experiment_name = config["experiment_name"]
78+
# exp = Experiment(workspace=ws, name=experiment_name)
79+
80+
run = Run(experiment=exp, run_id=run_id)
81+
names = run.get_file_names
82+
names()
83+
print("Run ID for last run: {}".format(run_id))
84+
model_local_dir = "model"
85+
os.makedirs(model_local_dir, exist_ok=True)
86+
87+
# Download Model to Project root directory
88+
model_name = "sklearn_regression_model.pkl"
89+
run.download_file(
90+
name="./outputs/" + model_name, output_file_path="./model/" + model_name
91+
)
92+
print("Downloaded model {} to Project root directory".format(model_name))
93+
os.chdir("./model")
94+
model = Model.register(
95+
model_path=model_name, # this points to a local file
96+
model_name=model_name, # this is the name the model is registered as
97+
tags={"area": "diabetes", "type": "regression", "run_id": run_id},
98+
description="Regression model for diabetes dataset",
99+
workspace=ws,
100+
)
101+
os.chdir("..")
102+
print(
103+
"Model registered: {} \nModel Description: {} \nModel Version: {}".format(
104+
model.name, model.description, model.version
105+
)
106+
)
107+
108+
# Remove the evaluate.json as we no longer need it
109+
# os.remove("aml_config/evaluate.json")
110+
111+
# Writing the registered model details to /aml_config/model.json
112+
model_json = {}
113+
model_json["model_name"] = model.name
114+
model_json["model_version"] = model.version
115+
model_json["run_id"] = run_id
116+
filename = "model_{}.json".format(args.config_suffix)
117+
output_path = os.path.join(args.json_config, filename)
118+
with open(output_path, "w") as outfile:
119+
json.dump(model_json, outfile)

‎code/scoring/create_scoring_image.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
"""
2+
Copyright (C) Microsoft Corporation. All rights reserved.​
3+
​
4+
Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual,
5+
royalty-free right to use, copy, and modify the software code provided by us
6+
("Software Code"). You may not sublicense the Software Code or any use of it
7+
(except to your affiliates and to vendors to perform work on your behalf)
8+
through distribution, network access, service agreement, lease, rental, or
9+
otherwise. This license does not purport to express any claim of ownership over
10+
data you may have shared with Microsoft in the creation of the Software Code.
11+
Unless applicable law gives you more rights, Microsoft reserves all other
12+
rights not expressly granted herein, whether by implication, estoppel or
13+
otherwise. ​
14+
​
15+
THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
16+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18+
MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
21+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22+
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23+
ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
24+
POSSIBILITY OF SUCH DAMAGE.
25+
"""
26+
import os, json, sys
27+
import argparse
28+
from azureml.core import Workspace
29+
from azureml.core.image import ContainerImage, Image
30+
from azureml.core import Run
31+
from azureml.core.model import Model
32+
from azureml.core.authentication import AzureCliAuthentication
33+
34+
cli_auth = AzureCliAuthentication()
35+
36+
run = Run.get_context()
37+
if "OfflineRun" in run.id:
38+
print("offline run")
39+
# Get workspace
40+
ws = Workspace.from_config(auth=cli_auth)
41+
else:
42+
exp = run.experiment
43+
ws = run.experiment.workspace
44+
45+
# Get the latest model details
46+
47+
parser = argparse.ArgumentParser("scoring_image")
48+
parser.add_argument(
49+
"--config_suffix", type=str, help="Datetime suffix for json config files"
50+
)
51+
parser.add_argument(
52+
"--json_config",
53+
type=str,
54+
help="Directory to write all the intermediate json configs",
55+
)
56+
args = parser.parse_args()
57+
58+
register_model_json = "model_{}.json".format(args.config_suffix)
59+
register_output_path = os.path.join(args.json_config, register_model_json)
60+
61+
62+
try:
63+
with open(register_output_path) as f:
64+
config = json.load(f)
65+
except:
66+
print("No new model to register thus no need to create new scoring image")
67+
# raise Exception('No new model to register as production model perform better')
68+
sys.exit(0)
69+
70+
model_name = config["model_name"]
71+
model_version = config["model_version"]
72+
73+
model_list = Model.list(workspace=ws)
74+
model, = (m for m in model_list if m.version == model_version and m.name == model_name)
75+
print(
76+
"Model picked: {} \nModel Description: {} \nModel Version: {}".format(
77+
model.name, model.description, model.version
78+
)
79+
)
80+
81+
os.chdir("./code/scoring")
82+
image_name = "diabetes-model-score"
83+
84+
image_config = ContainerImage.image_configuration(
85+
execution_script="score.py",
86+
runtime="python-slim",
87+
conda_file="conda_dependencies.yml",
88+
description="Image with ridge regression model",
89+
tags={"area": "diabetes", "type": "regression"},
90+
)
91+
92+
image = Image.create(
93+
name=image_name, models=[model], image_config=image_config, workspace=ws
94+
)
95+
96+
image.wait_for_creation(show_output=True)
97+
os.chdir("../..")
98+
99+
if image.creation_state != "Succeeded":
100+
raise Exception("Image creation status: {image.creation_state}")
101+
102+
print(
103+
"{}(v.{} [{}]) stored at {} with build log {}".format(
104+
image.name,
105+
image.version,
106+
image.creation_state,
107+
image.image_location,
108+
image.image_build_log_uri,
109+
)
110+
)
111+
112+
# Writing the image details to /aml_config/image.json
113+
image_json = {}
114+
image_json["image_name"] = image.name
115+
image_json["image_version"] = image.version
116+
image_json["image_location"] = image.image_location
117+
# with open("aml_config/image.json", "w") as outfile:
118+
# json.dump(image_json, outfile)
119+
filename = "image_{}.json".format(args.config_suffix)
120+
output_path = os.path.join(args.json_config, filename)
121+
with open(output_path, "w") as outfile:
122+
json.dump(image_json, outfile)
123+
124+
# How to fix the schema for a model, like if we have multiple models expecting different schema,

0 commit comments

Comments
 (0)