Skip to content

Commit e3dcd1a

Browse files
smlindauersmlindauer
authored
Merge pull request #118 from sassoftware/Git
New Feature: Git Integration
2 parents a0cf36b + 99233be commit e3dcd1a

File tree

3 files changed

+380
-3
lines changed

3 files changed

+380
-3
lines changed

‎src/sasctl/pzmm/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@
66
from .zipModel import ZipModel
77
from .writeScoreCode import ScoreCode
88
from .importModel import ImportModel
9+
from .gitIntegration import GitIntegrate

‎src/sasctl/pzmm/gitIntegration.py

Lines changed: 376 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,376 @@
1+
# Copyright (c) 2022, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
from msilib.schema import File
5+
from pathlib import Path
6+
from uuid import UUID
7+
from warnings import warn
8+
import zipfile
9+
from git import Repo
10+
import io
11+
12+
from .._services.model_repository import ModelRepository as mr
13+
from ..core import RestObj
14+
15+
16+
def getZippedModel(model, gPath, project=None):
17+
"""Retrieve a zipped file containing all of the model contents or a specified
18+
model. The project argument is only needed if the model argument is not a valid
19+
UUID or RestObj.
20+
21+
Parameters
22+
----------
23+
model : string or RestObj
24+
Model name, UUID, or RestObj which identifies the model. If only the model name
25+
is provided, the project name must also be supplied.
26+
gPath : string or Path
27+
Base directory of the git repository.
28+
project : string or RestObj, optional
29+
Project identifier, which is required when only the model name is supplied. Default
30+
is None.
31+
"""
32+
params = {"format": "zip"}
33+
modelZip = mr.get("models/%s" % (model), params=params, format_="content")
34+
modelName = mr.get_model(model).name
35+
# Check if the provided project variable is a REST object
36+
if isinstance(project, RestObj):
37+
projectName = project.name
38+
else:
39+
projectName = mr.get_project(project).name
40+
# Check to see if project folder exists
41+
if (Path(gPath) / projectName).exists():
42+
# Check to see if model folder exists
43+
if (Path(gPath) / projectName / modelName).exists():
44+
with open(
45+
Path(gPath) / projectName / modelName / (modelName + ".zip"), "wb"
46+
) as zFile:
47+
zFile.write(modelZip)
48+
else:
49+
newDir = Path(gPath) / projectName / modelName
50+
newDir.mkdir(parents=True, exist_ok=True)
51+
with open(
52+
Path(gPath) / projectName / modelName / (modelName + ".zip"), "wb"
53+
) as zFile:
54+
zFile.write(modelZip)
55+
else:
56+
newDir = Path(gPath) / projectName
57+
newDir.mkdir(parents=True, exist_ok=True)
58+
newDir = Path(gPath) / projectName / modelName
59+
newDir.mkdir(parents=True, exist_ok=True)
60+
with open(
61+
Path(gPath) / (projectName + "/" + modelName + ".zip"), "wb"
62+
) as zFile:
63+
zFile.write(modelZip)
64+
65+
return modelName, projectName
66+
67+
def project_exists(response, project):
68+
"""Checks if project exists on SAS Viya. If the project does not exist, then a new
69+
project is created or an error is raised.
70+
71+
Parameters
72+
----------
73+
response : RestObj
74+
JSON response of the get_project() call to model repository service.
75+
project : string or RestObj
76+
The name or id of the model project, or a RestObj representation of the project.
77+
78+
Returns
79+
-------
80+
response : RestObj
81+
JSON response of the get_project() call to model repository service.
82+
83+
Raises
84+
------
85+
SystemError
86+
Alerts user that API calls cannot continue until a valid project is provided.
87+
"""
88+
if response is None:
89+
try:
90+
warn("No project with the name or UUID {} was found.".format(project))
91+
UUID(project)
92+
raise SystemError(
93+
"The provided UUID does not match any projects found in SAS Model Manager. "
94+
+ "Please enter a valid UUID or a new name for a project to be created."
95+
)
96+
except ValueError:
97+
repo = mr.default_repository().get("id")
98+
response = mr.create_project(project, repo)
99+
print("A new project named {} was created.".format(response.name))
100+
return response
101+
else:
102+
return response
103+
104+
def model_exists(project, name, force):
105+
"""Checks if model already exists and either raises an error or deletes the redundant model.
106+
107+
Parameters
108+
----------
109+
project : string or dict
110+
The name or id of the model project, or a dictionary representation of the project.
111+
name : str or dict
112+
The name of the model.
113+
force : bool, optional
114+
Sets whether to overwrite models with the same name upon upload.
115+
116+
Raises
117+
------
118+
ValueError
119+
Model repository API cannot overwrite an already existing model with the upload model call.
120+
Alerts user of the force argument to allow multi-call API overwriting.
121+
"""
122+
project = mr.get_project(project)
123+
projectId = project["id"]
124+
projectModels = mr.get("/projects/{}/models".format(projectId))
125+
126+
for model in projectModels:
127+
# Throws a TypeError if only one model is in the project
128+
try:
129+
if model["name"] == name:
130+
if force:
131+
mr.delete_model(model.id)
132+
else:
133+
raise ValueError(
134+
"A model with the same model name exists in project {}. Include the force=True argument to overwrite models with the same name.".format(
135+
project.name
136+
)
137+
)
138+
except TypeError:
139+
if projectModels["name"] == name:
140+
if force:
141+
mr.delete_model(projectModels.id)
142+
else:
143+
raise ValueError(
144+
"A model with the same model name exists in project {}. Include the force=True argument to overwrite models with the same name.".format(
145+
project.name
146+
)
147+
)
148+
149+
150+
class GitIntegrate:
151+
@classmethod
152+
def pullViyaModel(
153+
cls,
154+
model,
155+
gPath,
156+
project=None,
157+
):
158+
"""Send an API request in order to pull a model from a project in
159+
SAS Model Manager in a zipped format. The contents of the zip file
160+
include all files found in SAS Model Manager's model UI, except that
161+
read-only json files are updated to match the current state of the model.
162+
163+
After pulling down the zipped model, unpack the file in the model folder.
164+
Overwrites files with the same name.
165+
166+
If supplying a model name instead of model UUID, a project name or uuid must
167+
be supplied as well. Models in the model repository are allowed duplicate
168+
names, therefore we need a method of parsing the returned models.
169+
170+
Parameters
171+
----------
172+
model : string or RestObj
173+
A string or JSON response representing the model to be pulled down
174+
gPath : string or Path
175+
Base directory of the git repository.
176+
project : string or RestObj, optional
177+
A string or JSON response representing the project the model exists in, default is None.
178+
"""
179+
# Try to pull down the model assuming a UUID or RestObj is provided
180+
try:
181+
if isinstance(model, RestObj):
182+
model = model.id
183+
else:
184+
UUID(model)
185+
projectName = mr.get_model(model).projectName
186+
modelName, projectName = getZippedModel(model, gPath, projectName)
187+
# If a name is provided instead, use the provided project name or UUID to find the correct model
188+
except ValueError:
189+
projectResponse = mr.get_project(project)
190+
if projectResponse is None:
191+
raise SystemError(
192+
"For models with only a provided name, a project name or UUID must also be supplied."
193+
)
194+
projectName = projectResponse["name"]
195+
projectId = projectResponse["id"]
196+
projectModels = mr.get("/projects/{}/models".format(projectId))
197+
for model in projectModels:
198+
# Throws a TypeError if only one model is in the project
199+
try:
200+
if model["name"] == model:
201+
modelId = model.id
202+
modelName, projectName = getZippedModel(
203+
modelId, gPath, projectName
204+
)
205+
except TypeError:
206+
if projectModels["name"] == model:
207+
modelId = projectModels.id
208+
modelName, projectName = getZippedModel(
209+
modelId, gPath, projectName
210+
)
211+
212+
# Unpack the pulled down zip model and overwrite any duplicate files
213+
mPath = Path(gPath) / "{projectName}/{modelName}".format(
214+
projectName=projectName, modelName=modelName
215+
)
216+
with zipfile.ZipFile(str(mPath / (modelName + ".zip")), mode="r") as zFile:
217+
zFile.extractall(str(mPath))
218+
219+
# Delete the zip model objects in the directory to minimize confusion when uploading back to SAS Model Manager
220+
for zipFile in mPath.glob("*.zip"):
221+
zipFile.unlink()
222+
223+
@classmethod
224+
def pushGitModel(cls, gPath, modelName=None, projectName=None):
225+
"""Push a single model in the git repository up to SAS Model Manager. This function
226+
creates an archive of all files in the directory and imports the zipped model.
227+
228+
Parameters
229+
----------
230+
gPath : string or Path
231+
Base directory of the git repository or path which includes project and model directories.
232+
modelName : string, optional
233+
Name of model to be imported, by default None
234+
projectName : string, optional
235+
Name of project the model is imported from, by default None
236+
"""
237+
if modelName is None and projectName is None:
238+
modelDir = gPath
239+
modelName = modelDir.name
240+
projectName = modelDir.parent.name
241+
else:
242+
modelDir = Path(gPath) / (projectName + "/" + modelName)
243+
for zipFile in modelDir.glob("*.zip"):
244+
zipFile.unlink()
245+
fileNames = []
246+
fileNames.extend(sorted(Path(modelDir).glob("*")))
247+
with zipfile.ZipFile(
248+
str(modelDir / (modelDir.name + ".zip")), mode="w"
249+
) as zFile:
250+
for file in fileNames:
251+
zFile.write(str(file), arcname=file.name)
252+
with open(modelDir / (modelDir.name + ".zip"), "rb") as zFile:
253+
zipIOFile = io.BytesIO(zFile.read())
254+
# Check if model with same name already exists in project. Delete if it exists.
255+
model_exists(projectName, modelName, True)
256+
mr.import_model_from_zip(modelName, projectName, zipIOFile)
257+
258+
@classmethod
259+
def gitRepoPush(cls, gPath, commitMessage, branch="origin"):
260+
"""Create a new commit with new files, then push changes from the local repository to a remote
261+
branch. The default remote branch is origin.
262+
263+
Parameters
264+
----------
265+
gPath : string or Path
266+
Base directory of the git repository.
267+
commitMessage : string
268+
Commit message for the new commit
269+
branch : str, optional
270+
Branch name for the remote repository, by default 'origin'
271+
"""
272+
repo = Repo(gPath)
273+
repo.git.add(all=True)
274+
repo.index.commit(commitMessage)
275+
pushBranch = repo.remote(name=branch)
276+
pushBranch.push()
277+
278+
@classmethod
279+
def gitRepoPull(cls, gPath, branch="origin"):
280+
"""Pull down any changes from a remote branch of the git repository. The default branch is
281+
origin.
282+
283+
Parameters
284+
----------
285+
gPath : string or Path
286+
Base directory of the git repository.
287+
branch : string
288+
Branch name for the remote repository, by default 'origin'
289+
"""
290+
repo = Repo(gPath)
291+
pullBranch = repo.remote(name=branch)
292+
pullBranch.pull()
293+
294+
@classmethod
295+
def pullGitProject(cls, gPath, project=None):
296+
"""Using a user provided project name, search for the project in the specified git repository,
297+
check if the project already exists on SAS Model Manager (create a new project if it does not),
298+
then upload each model found in the git project to SAS Model Manager
299+
300+
Parameters
301+
----------
302+
gPath : string or Path
303+
Base directory of the git repository or the project directory.
304+
project : string or RestObj
305+
Project name, UUID, or JSON response from SAS Model Manager.
306+
"""
307+
# Check to see if provided project argument is a valid project on SAS Model Manager
308+
projectResponse = mr.get_project(project)
309+
project = project_exists(projectResponse, project)
310+
projectName = project.name
311+
312+
# Check if project exists in git path and produce an error if it does not
313+
pPath = Path(gPath) / projectName
314+
if pPath.exists():
315+
models = [x for x in pPath.glob("*") if x.is_dir()]
316+
if len(models) == 0:
317+
print("No models were found in project {}.".format(projectName))
318+
print(
319+
"{numModels} models were found in project {projectName}.".format(
320+
numModels=len(models), projectName=projectName
321+
)
322+
)
323+
else:
324+
raise FileNotFoundError(
325+
"No directory with the name {} was found in the specified git path.".format(
326+
project
327+
)
328+
)
329+
330+
# Loop through paths of models and upload each to SAS Model Manager
331+
for model in models:
332+
# Remove any extra zip objects in the directory
333+
for zipFile in model.glob("*.zip"):
334+
zipFile.unlink()
335+
cls.pushGitModel(model)
336+
337+
@classmethod
338+
def pullMMProject(cls, gPath, project):
339+
"""Following the user provided project argument, pull down all models from the
340+
corresponding SAS Model Manager project into the mapped git directories.
341+
342+
Parameters
343+
----------
344+
gPath : string or Path
345+
Base directory of the git repository.
346+
project : string or RestObj
347+
The name or id of the model project, or a RestObj representation of the project.
348+
"""
349+
# Check to see if provided project argument is a valid project on SAS Model Manager
350+
projectResponse = mr.get_project(project)
351+
project = project_exists(projectResponse, project)
352+
projectName = project.name
353+
# Check if project exists in git path and create it if it does not
354+
pPath = Path(gPath) / projectName
355+
if not pPath.exists():
356+
Path(pPath).mkdir(parents=True, exist_ok=True)
357+
358+
# Return a list of model names from SAS Model Manager project
359+
modelResponse = mr.get("projects/{}/models".format(project.id))
360+
if modelResponse == []:
361+
raise FileNotFoundError(
362+
"No models were found in the specified project. A new project folder "
363+
+ "has been created if it did not already exist within the git repository."
364+
)
365+
modelNames = []
366+
modelId = []
367+
for model in modelResponse:
368+
modelNames.append(model.name)
369+
modelId.append(model.id)
370+
# For each model, search for an appropriate model directory in the project directory and pull down the model
371+
for name, id in zip(modelNames, modelId):
372+
mPath = pPath / name
373+
# If the model directory does not exist, create one in the project directory
374+
if not mPath.exists():
375+
Path(mPath).mkdir(parents=True, exist_ok=True)
376+
cls.pullViyaModel(id, mPath.parents[1])

0 commit comments

Comments
 (0)