Skip to content

Commit 6db9258

Browse files
ciaran28ciaran28
committed
Development
1 parent a4ffd9d commit 6db9258

File tree

8 files changed

+114
-27
lines changed

8 files changed

+114
-27
lines changed

‎mlOps/devOps/params/development/repos.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
{
1010
"url": "https://github.com/ciaran28/dstoolkit-mlops-databricks",
1111
"provider": "gitHub",
12-
"path": "DevelopmentFolder"
12+
"path": "DevelopmentFolder",
13+
"branch": "main"
1314
}
1415
]
1516
}

‎mlOps/devOps/params/production/repos.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
{
1010
"url": "https://github.com/ciaran28/dstoolkit-mlops-databricks",
1111
"provider": "gitHub",
12-
"path": "ProductionFolder"
12+
"path": "ProductionFolder",
13+
"branch": "main"
1314
}
1415
]
1516
}

‎mlOps/devOps/params/sandbox/repos.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
{
1010
"url": "https://github.com/ciaran28/dstoolkit-mlops-databricks",
1111
"provider": "gitHub",
12-
"path": "Sandbox"
12+
"path": "Sandbox",
13+
"branch": "main"
1314
}
1415
]
1516
}

‎mlOps/devOps/params/uat/repos.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
],
88
"Repo_Configuration": [
99
{
10-
"url": "https://github.com/ciaran28/dstoolkit-mlops-databricks",
11-
"provider": "gitHub",
12-
"path": "UATFolder"
10+
"url": "https://github.com/ciaran28/dstoolkit-mlops-databricks",
11+
"provider": "gitHub",
12+
"path": "UATFolder",
13+
"branch": "main"
1314
}
1415
]
1516
}

‎mlOps/devOps/utils/python/__pycache__/utils_repo_pull.cpython-310.pyc‎

2.65 KB
Binary file not shown.

‎mlOps/devOps/utils/python/utils_repo_pull.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ def update_repo(repo_id, update_branch):
5555
Invoked Databricks API to update repo
5656
"""
5757

58+
repo_id = str(repo_id)
59+
5860
WORKSPACE_ID = os.environ.get("WORKSPACE_ID")
5961
DATABRICKS_INSTANCE = os.environ.get("DATABRICKS_INSTANCE")
6062
DATABRICKS_AAD_TOKEN = os.environ.get("DATABRICKS_AAD_TOKEN")
@@ -97,20 +99,27 @@ def main():
9799
print(f"Repos To Connect {repo_param_file}")
98100

99101
repos_with_management_permissions, status_code = get_repos_with_management_permissions()
100-
102+
101103
for repo in repo_param_file:
104+
105+
102106
update_folder = repo['path']
103107
update_branch = repo['branch']
108+
104109
for item in repos_with_management_permissions:
105110
print(f" The Update Folder is {update_folder} and path is {item['path']}")
111+
106112
if update_folder in item['path']:
107113
print(f" The Update Folder {update_folder} is Contained within the Path {item['path']}")
108114
print("Retrieve the Repo ID")
109115

110116
repo_id = str(item['id'])
111117

112118
#Update repo
119+
#import pdb; pdb.set_trace()
113120
status_code = update_repo(repo_id, update_branch)
121+
122+
return status_code
114123

115124

116125
if __name__ == "__main__":

‎mlOps/devOps/utils/tests/__pycache__/test_utils_repo_pull.cpython-310-pytest-7.3.1.pyc‎

7.25 KB
Binary file not shown.

‎mlOps/devOps/utils/tests/test_utils_repo_pull.py‎

Lines changed: 94 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import json
77
import requests
88

9-
from python.utils_repo_pull import _ingest_repo_param_file, get_repos_with_management_permissions, update_repo
9+
from python.utils_repo_pull import _ingest_repo_param_file, get_repos_with_management_permissions, update_repo, main
1010

1111

1212
class TestIngestRepoParamFile(unittest.TestCase):
@@ -52,7 +52,7 @@ def test_load_json(self, mock_open):
5252
class GetReposWithManagementPermissions(unittest.TestCase):
5353

5454
@patch('requests.get')
55-
def test_get_repos_with_management_permissions_success(mock_get):
55+
def test_get_repos_with_management_permissions_success(self, mock_get):
5656
monkeypatch = MonkeyPatch()
5757

5858
monkeypatch.setenv('ARM_CLIENT_ID', 'test_arm_client_id')
@@ -72,15 +72,7 @@ def test_get_repos_with_management_permissions_success(mock_get):
7272
"provider":"gitHub",
7373
"branch":"main",
7474
"head_commit_id":"test_commit_id"
75-
},
76-
{
77-
"id":2806425392675498,
78-
"path":"/Repos/***/test_dbx_repo_folder_two",
79-
"url":"https://github.com/test_repo_profile/test_repo_two",
80-
"provider":"gitHub",
81-
"branch":"main",
82-
"head_commit_id":"test_commit_id"
83-
}
75+
}
8476
]
8577
}
8678

@@ -98,11 +90,12 @@ def test_get_repos_with_management_permissions_success(mock_get):
9890
'X-Databricks-Azure-Workspace-Resource-Id': 'test_workspace_id',
9991
'Content-Type': 'application/json'}
10092

101-
mock_get.assert_once_called_with(
102-
"https://test_databricks_instance/api/2.0/repos",
93+
94+
mock_get.assert_called_once_with(
95+
'https://test_databricks_instance/api/2.0/repos',
10396
headers=expected_dbkrs_req_headers
104-
)
105-
97+
)
98+
10699

107100
@patch('requests.get')
108101
def test_get_repos_with_management_permissions_failure(mock_get):
@@ -125,8 +118,8 @@ def test_get_repos_with_management_permissions_failure(mock_get):
125118

126119
class UpdateRepo(unittest.TestCase):
127120

128-
@patch('requests.post')
129-
def test_update_repo_success(mock_post):
121+
@patch('requests.patch')
122+
def test_update_repo_success(self, mock_patch):
130123
monkeypatch = MonkeyPatch()
131124
monkeypatch.setenv('ARM_CLIENT_ID', 'test_arm_client_id')
132125
monkeypatch.setenv('WORKSPACE_ID', 'test_workspace_id')
@@ -137,7 +130,7 @@ def test_update_repo_success(mock_post):
137130
mock_repo_id = 123456789
138131
mock_update_branch = "test_main_branch"
139132

140-
mock_post.return_value.status_code = 200
133+
mock_patch.return_value.status_code = 200
141134

142135
status_code = update_repo(mock_repo_id, mock_update_branch )
143136

@@ -150,15 +143,16 @@ def test_update_repo_success(mock_post):
150143
'X-Databricks-Azure-Workspace-Resource-Id': 'test_workspace_id',
151144
'Content-Type': 'application/json'}
152145

153-
mock_post.assert_once_called_with(
146+
mock_patch.assert_called_once_with(
154147
"https://test_databricks_instance/api/2.0/repos/" + str(mock_repo_id),
155148
headers=expected_dbkrs_req_headers,
156149
json={
157150
"branch": mock_update_branch
158151
}
159152
)
160153

161-
def test_update_repo_failure(mock_post):
154+
@patch('requests.post')
155+
def test_update_repo_failure(self, mock_post):
162156

163157
mock_repo_id = 123456789
164158
mock_update_branch = "test_main_branch"
@@ -170,6 +164,86 @@ def test_update_repo_failure(mock_post):
170164
assert status_code == 500
171165

172166

167+
class Main(unittest.TestCase):
168+
169+
test_repo_json = {
170+
"Git_Configuration": [
171+
{
172+
"git_username": "test_username",
173+
"git_provider": "test_provider",
174+
}
175+
],
176+
"Repo_Configuration": [
177+
{
178+
"url": "test_url",
179+
"provider": "test_provider",
180+
"path": "test_folder"
181+
}
182+
]
183+
}
184+
test_repo_json = json.dumps(test_repo_json)
185+
186+
187+
@patch('python.utils_repo_pull.update_repo')
188+
@patch('python.utils_repo_pull.get_repos_with_management_permissions')
189+
@patch('python.utils_repo_pull._ingest_repo_param_file')
190+
def test_main_success(self, mock_ingest_repo_param_file, mock_get_repos_with_management_permissions, mock_update_repo):
191+
192+
# monkey patch environment variables
193+
monkeypatch = MonkeyPatch()
194+
monkeypatch.setenv('ENVIRONMENT', 'test_environment')
195+
monkeypatch.setenv('ARM_CLIENT_ID', 'test_arm_client_id')
196+
monkeypatch.setenv('WORKSPACE_ID', 'test_workspace_id')
197+
monkeypatch.setenv('DATABRICKS_MANAGEMENT_TOKEN', 'test_databricks_management_token')
198+
monkeypatch.setenv('DATABRICKS_AAD_TOKEN', 'test_databricks_aad_token')
199+
monkeypatch.setenv('DATABRICKS_INSTANCE', 'test_databricks_instance')
200+
201+
202+
mock_ingest_repo_param_file_json_return = [{
203+
"url": "test_url",
204+
"provider": "test_provider",
205+
"path": "test_folder",
206+
"branch": "test_branch"
207+
}]
208+
209+
mock_ingest_repo_param_file.return_value = mock_ingest_repo_param_file_json_return
210+
211+
# Should be doing a mock open instead !!!
212+
#mock_ingest_repo_param_file.return_value = mock_ingest_repo_param_file_json_return
213+
214+
# mock return value from get repos with management permissions
215+
mock_get_repos_with_management_permissions_json_return = [
216+
{
217+
"id":61449681029719,
218+
"path":"/Repos/***/test_folder",
219+
"url":"https://github.com/test_repo_profile/test_repo_one",
220+
"provider":"gitHub",
221+
"branch":"main",
222+
"head_commit_id":"test_commit_id"
223+
}
224+
]
225+
226+
mock_get_repos_with_management_permissions.return_value = (mock_get_repos_with_management_permissions_json_return, 200)
227+
228+
# mock return value from update repo
229+
mock_update_repo.return_value = 200
230+
231+
# call main function
232+
status_code = main()
233+
234+
# assert main function returns 200
235+
assert status_code == 200
236+
237+
238+
# assert mock functions were called using correct arguments
239+
mock_ingest_repo_param_file.assert_called_once_with('mlOps/devOps/params/test_environment/repos.json')
240+
mock_update_repo.assert_called_once_with("61449681029719", "test_branch")
241+
242+
243+
244+
245+
246+
173247

174248

175249

0 commit comments

Comments
 (0)