Skip to content

Commit b86c5a0

Browse files
smlindauersmlindauer
committed
First draft of git integration tools
1 parent 1156c6e commit b86c5a0

File tree

1 file changed

+289
-0
lines changed

1 file changed

+289
-0
lines changed

‎src/sasctl/pzmm/gitIntegration.py

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
# Copyright (c) 2022, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
from msilib.schema import File
5+
from pathlib import Path
6+
from uuid import UUID
7+
from warnings import warn
8+
import shutil
9+
from git import Repo
10+
import io
11+
12+
from .._services.model_repository import ModelRepository as mr
13+
from ..core import RestObj
14+
15+
def getZippedModel(model, gPath, project=None):
16+
'''Retrieve a zipped file containing all of the model contents or a specified
17+
model. The project argument is only needed if the model argument is not a valid
18+
UUID or RestObj.
19+
20+
Parameters
21+
----------
22+
model : string or RestObj
23+
Model name, UUID, or RestObj which identifies the model. If only the model name
24+
is provided, the project name must also be supplied.
25+
gPath : string or Path
26+
Base directory of the git repository.
27+
project : string or RestObj, optional
28+
Project identifier, which is required when only the model name is supplied. Default
29+
is None.
30+
'''
31+
params = {'format': 'zip'}
32+
modelZip = mr.get('models/%s' % (model), params=params, format_='content')
33+
modelName = mr.get_model(model).name
34+
# Check if the provided project variable is a REST object
35+
if isinstance(project, RestObj):
36+
projectName = project.name
37+
else:
38+
projectName = mr.get_project(project).name
39+
# Attempt to put model in project folder
40+
try:
41+
with open(Path(gPath) / (projectName + '/' + modelName + '.zip'), 'wb') as zFile:
42+
zFile.write(modelZip)
43+
# If the folder doesn't exist, create it and then put model into the project folder
44+
except FileNotFoundError:
45+
newDir = Path(projectName + '/')
46+
newDir.mkdir(parents=True, exist_ok=True)
47+
with open(Path(gPath) / (projectName + '/' + modelName + '.zip'), 'wb') as zFile:
48+
zFile.write(modelZip)
49+
return modelName, projectName
50+
51+
def project_exists(response, project):
52+
"""Checks if project exists on SAS Viya. If the project does not exist, then a new
53+
project is created or an error is raised.
54+
55+
Parameters
56+
----------
57+
response : RestObj
58+
JSON response of the get_project() call to model repository service.
59+
project : string or RestObj
60+
The name or id of the model project, or a RestObj representation of the project.
61+
62+
Returns
63+
-------
64+
response : RestObj
65+
JSON response of the get_project() call to model repository service.
66+
67+
Raises
68+
------
69+
SystemError
70+
Alerts user that API calls cannot continue until a valid project is provided.
71+
"""
72+
if response is None:
73+
try:
74+
warn("No project with the name or UUID {} was found.".format(project))
75+
UUID(project)
76+
raise SystemError(
77+
"The provided UUID does not match any projects found in SAS Model Manager. "
78+
+ "Please enter a valid UUID or a new name for a project to be created."
79+
)
80+
except ValueError:
81+
repo = mr.default_repository().get("id")
82+
response = mr.create_project(project, repo)
83+
print("A new project named {} was created.".format(response.name))
84+
return response
85+
else:
86+
return response
87+
88+
class GitIntegrate:
89+
@classmethod
90+
def pullViyaModel(
91+
cls,
92+
model,
93+
gPath,
94+
project=None,
95+
):
96+
'''Send an API request in order to pull a model from a project in
97+
SAS Model Manager in a zipped format. The contents of the zip file
98+
include all files found in SAS Model Manager's model UI, except that
99+
read-only json files are updated to match the current state of the model.
100+
101+
After pulling down the zipped model, unpack the file in the model folder.
102+
Overwrites files with the same name.
103+
104+
If supplying a model name instead of model UUID, a project name or uuid must
105+
be supplied as well. Models in the model repository are allowed duplicate
106+
names, therefore we need a method of parsing the returned models.
107+
108+
Parameters
109+
----------
110+
model : string or RestObj
111+
A string or JSON response representing the model to be pulled down
112+
gPath : string or Path
113+
Base directory of the git repository.
114+
project : string or RestObj, optional
115+
A string or JSON response representing the project the model exists in, default is None.
116+
'''
117+
# Try to pull down the model assuming a UUID or RestObj is provided
118+
try:
119+
if isinstance(model, RestObj):
120+
model = model.id
121+
else:
122+
UUID(model)
123+
projectName = mr.get_model(model).projectName
124+
modelName = getZippedModel(model, projectName, gPath)
125+
# If a name is provided instead, use the provided project name or UUID to find the correct model
126+
except ValueError:
127+
projectResponse = mr.get_project(project)
128+
if projectResponse is None:
129+
raise SystemError(
130+
"For models with only a provided name, a project name or UUID must also be supplied."
131+
)
132+
projectName = projectResponse["name"]
133+
projectId = projectResponse["id"]
134+
projectModels = mr.get("/projects/{}/models".format(projectId))
135+
for model in projectModels:
136+
# Throws a TypeError if only one model is in the project
137+
try:
138+
if model["name"] == model:
139+
modelId = model.id
140+
modelName = getZippedModel(modelId, projectName, gPath)
141+
except TypeError:
142+
if projectModels["name"] == model:
143+
modelId = projectModels.id
144+
modelName = getZippedModel(modelId, projectName, gPath)
145+
146+
# Unpack the pulled down zip model and overwrite any duplicate files
147+
mPath = Path(gPath) / '{projectName}/{modelName}'.format(projectName=projectName, modelName=modelName)
148+
shutil.unpack_archive(filename=(modelName + '.zip'), extract_dir=mPath)
149+
150+
# Delete the zip model objects in the directory to minimize confusion when uploading back to SAS Model Manager
151+
for zipFile in mPath.glob('*.zip'):
152+
zipFile.unlink()
153+
154+
@classmethod
155+
def pushGitModel(cls, gPath, modelName=None, projectName=None):
156+
'''Push a single model in the git repository up to SAS Model Manager. This function
157+
creates an archive of all files in the directory and imports the zipped model.
158+
159+
Parameters
160+
----------
161+
gPath : string or Path
162+
Base directory of the git repository.
163+
modelName : string, optional
164+
Name of model to be imported, by default None
165+
projectName : string, optional
166+
Name of project the model is imported from, by default None
167+
'''
168+
if modelName is None and projectName is None:
169+
modelDir = gPath
170+
else:
171+
modelDir = Path(gPath) / (projectName + '/' + modelName)
172+
for zipFile in modelDir.glob('*.zip'):
173+
zipFile.unlink()
174+
shutil.make_archive(modelName, 'zip', modelDir)
175+
with open(modelDir / (modelName + '.zip'), 'rb') as zFile:
176+
zipIOFile = io.BytesIO(zFile.read())
177+
mr.import_model_from_zip(modelName, projectName, zipIOFile)
178+
179+
@classmethod
180+
def gitRepoPush(cls, gPath, commitMessage, branch='origin'):
181+
'''Create a new commit with new files, then push changes from the local repository to a remote
182+
branch. The default remote branch is origin.
183+
184+
Parameters
185+
----------
186+
gPath : string or Path
187+
Base directory of the git repository.
188+
commitMessage : string
189+
Commit message for the new commit
190+
branch : str, optional
191+
Branch name for the remote repository, by default 'origin'
192+
'''
193+
repo = Repo(gPath)
194+
repo.git.add(update=True)
195+
repo.index.commit(commitMessage)
196+
pushBranch = repo.remote(name=branch)
197+
pushBranch.push()
198+
199+
@classmethod
200+
def gitRepoPull(cls, gPath, branch='origin'):
201+
'''Pull down any changes from a remote branch of the git repository. The default branch is
202+
origin.
203+
204+
Parameters
205+
----------
206+
gPath : string or Path
207+
Base directory of the git repository.
208+
branch : string
209+
Branch name for the remote repository, by default 'origin'
210+
'''
211+
repo = Repo(gPath)
212+
pullBranch = repo.remote(name=branch)
213+
pullBranch.pull()
214+
215+
@classmethod
216+
def pullGitProject(cls, gPath, project=None):
217+
'''Using a user provided project name, search for the project in the specified git repository,
218+
check if the project already exists on SAS Model Manager (create a new project if it does not),
219+
then upload each model found in the git project to SAS Model Manager
220+
221+
Parameters
222+
----------
223+
gPath : string or Path
224+
Base directory of the git repository or the project directory.
225+
project : string or RestObj
226+
Project name, UUID, or JSON response from SAS Model Manager.
227+
'''
228+
# Check to see if provided project argument is a valid project on SAS Model Manager
229+
projectResponse = mr.get_project(project)
230+
project = project_exists(projectResponse, project)
231+
projectName = project.name
232+
233+
# Check if project exists in git path and produce an error if it does not
234+
pPath = Path(gPath) / projectName
235+
if pPath.exists():
236+
models = [x for x in pPath.glob('*') if x.is_dir()]
237+
if len(models) == 0:
238+
print('No models were found in project {}.'.format(projectName))
239+
print('{numModels} were found in project {projectName}.'.format(numModels=len(models), projectName=projectName))
240+
else:
241+
raise FileNotFoundError('No directory with the name {} was found in the specified git path.'.format(project))
242+
243+
# Loop through paths of models and upload each to SAS Model Manager
244+
for model in models:
245+
# Remove any extra zip objects in the directory
246+
for zipFile in model.glob('*.zip'):
247+
zipFile.unlink()
248+
cls.pushGitModel(model)
249+
250+
251+
@classmethod
252+
def pullMMProject(cls, gPath, project):
253+
'''Following the user provided project argument, pull down all models from the
254+
corresponding SAS Model Manager project into the mapped git directories.
255+
256+
Parameters
257+
----------
258+
gPath : string or Path
259+
Base directory of the git repository.
260+
project : string or RestObj
261+
The name or id of the model project, or a RestObj representation of the project.
262+
'''
263+
# Check to see if provided project argument is a valid project on SAS Model Manager
264+
projectResponse = mr.get_project(project)
265+
project = project_exists(projectResponse, project)
266+
projectName = project.name
267+
268+
# Check if project exists in git path and create it if it does not
269+
pPath = Path(gPath) / projectName
270+
if not pPath.exists():
271+
Path(pPath).mkdir(parents=True, exist_ok=True)
272+
273+
# Return a list of model names from SAS Model Manager project
274+
modelResponse = mr.get('projects/{}/models'.format(project.id))
275+
if modelResponse == []:
276+
raise FileNotFoundError('No models were found in the specified project. A new project folder ' +
277+
'has been created if it did not already exist within the git repository.')
278+
modelNames, modelId = []*2
279+
for i, model in enumerate(modelResponse):
280+
modelNames[i] = model.name
281+
modelId[i] = model.id
282+
283+
# For each model, search for an appropriate model directory in the project directory and pull down the model
284+
for name, id in zip(modelNames, modelId):
285+
mPath = pPath / name
286+
# If the model directory does not exist, create one in the project directory
287+
if not mPath.exists():
288+
Path(mPath).mkdir(parents=True, exist_ok=True)
289+
cls.pullViyaModel(id, mPath.parents[1])

0 commit comments

Comments
 (0)