Skip to content
This repository was archived by the owner on Jan 14, 2021. It is now read-only.

Commit e2d5289

Browse files
mprahlmprahl
committed
Add the audit function to the RetireComponentHandler handler
1 parent e132ab5 commit e2d5289

File tree

2 files changed

+314
-31
lines changed

2 files changed

+314
-31
lines changed

‎pdcupdater/handlers/retirement.py‎

Lines changed: 89 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import logging
22
from datetime import datetime
3-
43
import requests
4+
55
import pdcupdater.services
66

77
log = logging.getLogger(__name__)
@@ -39,11 +39,7 @@ def handle(self, pdc, msg):
3939
branch = msg['msg']['commit']['branch']
4040
repo = msg['msg']['commit']['repo']
4141
namespace = msg['msg']['commit']['namespace']
42-
# The dist-git namespaces are plural but the types are singular in PDC
43-
if namespace.endswith('s'):
44-
component_type = namespace[:-1]
45-
else:
46-
component_type = namespace
42+
component_type = self._namespace_to_pdc(namespace)
4743
# This query guarantees a unique component branch, so a count of 1 is
4844
# expected
4945
branch_query_rv = pdc['component-branches']._(
@@ -63,22 +59,93 @@ def handle(self, pdc, msg):
6359
@staticmethod
6460
def _retire_branch(pdc, branch):
6561
""" Internal method for retiring a branch in PDC. """
66-
log.info("Retiring {type}/{global_component}#{name}".format(**branch))
6762
today = datetime.utcnow().date()
6863
for sla in branch['slas']:
6964
sla_eol = datetime.strptime(sla['eol'], '%Y-%m-%d').date()
7065
if sla_eol > today:
7166
pdc['component-branch-slas'][sla['id']]._ \
7267
+= {'eol': str(today)}
7368

69+
@staticmethod
70+
def _namespace_to_pdc(namespace):
71+
namespace_to_pdc = {
72+
'rpms': 'rpm',
73+
'modules': 'module',
74+
'container': 'container',
75+
}
76+
if namespace not in namespace_to_pdc:
77+
raise ValueError('The namespace "{0}" is not supported'
78+
.format(namespace))
79+
else:
80+
return namespace_to_pdc[namespace]
81+
82+
@staticmethod
83+
def _pdc_to_namespace(pdc_type):
84+
pdc_to_namespace = {
85+
'rpm': 'rpms',
86+
'module': 'modules',
87+
'container': 'container',
88+
}
89+
if pdc_type not in pdc_to_namespace:
90+
raise ValueError('The PDC type "{0}" is not supported'
91+
.format(pdc_type))
92+
else:
93+
return pdc_to_namespace[pdc_type]
94+
95+
@staticmethod
96+
def _is_retired_in_cgit(namespace, repo, branch, requests_session=None):
97+
if requests_session is None:
98+
requests_session = requests.Session()
99+
100+
cgit_url = 'https://src.fedoraproject.org/cgit'
101+
# Check to see if they have a dead.package file in dist-git
102+
url = '{base}/{namespace}/{repo}.git/plain/dead.package?h={branch}'
103+
response = requests_session.head(url.format(
104+
base=cgit_url,
105+
namespace=namespace,
106+
repo=repo,
107+
branch=branch,
108+
))
109+
110+
# If there is a dead.package, then the branch is retired in cgit
111+
if response.status_code in [200, 404]:
112+
return response.status_code == 200
113+
else:
114+
raise ValueError(
115+
'The connection to cgit failed. Retirement status could not '
116+
'be determined. The status code was: {0}. The content was: '
117+
'{1}'.format(response.status_code, response.content))
118+
74119
def audit(self, pdc):
75-
""" Not Implemented.
120+
""" Returns the difference in retirement status in PDC and dist-git.
76121
77-
This function (if it were implemented) should compare the status in PDC
78-
and the status in the "real world" (i.e., in dist-git) and return the
79-
difference.
122+
This function compares the status in PDC and the status in the
123+
"real world" (i.e., in dist-git) and return the difference.
80124
"""
81-
pass
125+
branches_retired_in_distgit = set()
126+
branches_retired_in_pdc = set()
127+
session = requests.Session()
128+
129+
log.info('Looking up all branches from PDC.')
130+
for branch in pdc.get_paged(pdc['component-branches']._):
131+
branch_str = '{type}/{global_component}#{name}'.format(**branch)
132+
log.debug('Considering {0}'.format(branch_str))
133+
retired_in_cgit = self._is_retired_in_cgit(
134+
namespace=self._pdc_to_namespace(branch['type']),
135+
repo=branch['global_component'],
136+
branch=branch['name'],
137+
requests_session=session
138+
)
139+
140+
if retired_in_cgit:
141+
branches_retired_in_distgit.add(branch_str)
142+
if not branch['active']:
143+
branches_retired_in_pdc.add(branch_str)
144+
145+
present = branches_retired_in_pdc - branches_retired_in_distgit
146+
absent = branches_retired_in_distgit - branches_retired_in_pdc
147+
148+
return present, absent
82149

83150
def initialize(self, pdc):
84151
""" Initialize PDC retirement status from analyzing dist-git.
@@ -87,28 +154,19 @@ def initialize(self, pdc):
87154
in PDC that have a dead.package file in dist-git.
88155
"""
89156
session = requests.Session()
90-
cgit_url = "https://src.fedoraproject.org/cgit"
91-
pdc2namespace = {
92-
'rpm': 'rpms',
93-
'module': 'modules',
94-
'container': 'container',
95-
}
96157

97158
# Look up all non-retired branches from PDC
98-
log.info("Looking up active branches from PDC.")
99-
branches = pdc.get_paged(pdc['component-branches'], active=True)
100-
101-
for branch in branches:
102-
log.debug("Considering {type}/{global_component}#{name}".format(**branch))
103-
# Check to see if they have a dead.package file in dist-git
104-
url = "{base}/{type}/{repo}.git/plain/dead.package?h={branch}"
105-
response = session.head(url.format(
106-
base=cgit_url,
107-
type=pdc2namespace[branch['type']],
159+
log.info('Looking up active branches from PDC.')
160+
161+
for branch in pdc.get_paged(pdc['component-branches']._, active=True):
162+
log.debug('Considering {type}/{global_component}#{name}'
163+
.format(**branch))
164+
retired_in_cgit = self._is_retired_in_cgit(
165+
namespace=self._pdc_to_namespace(branch['type']),
108166
repo=branch['global_component'],
109167
branch=branch['name'],
110-
))
168+
requests_session=session
169+
)
111170

112-
# If so, then we need to retire them.
113-
if bool(response):
171+
if retired_in_cgit:
114172
self._retire_branch(pdc, branch)

‎pdcupdater/tests/handler_tests/test_retirement.py‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
import mock
2+
13
from pdcupdater.tests.handler_tests import BaseHandlerTest, mock_pdc
24
import pdcupdater.services
5+
from pdcupdater.handlers.retirement import RetireComponentHandler
36

47

58
class TestRetiredComponents(BaseHandlerTest):
@@ -114,3 +117,225 @@ def test_can_process_retire_msg_already_retired(self, pdc):
114117
'component-branches'
115118
]
116119
self.assertEquals(pdc.calls.keys(), expected_keys)
120+
121+
@mock_pdc
122+
def test_audit(self, pdc):
123+
pdc.add_endpoint('component-branches', 'GET', [
124+
{
125+
"id": 155867,
126+
"global_component": "obexftp",
127+
"name": "f26",
128+
"slas": [
129+
{
130+
"id": 310591,
131+
"sla": "bug_fixes",
132+
"eol": "2017-06-28"
133+
},
134+
{
135+
"id": 310602,
136+
"sla": "security_fixes",
137+
"eol": "2017-06-28"
138+
}
139+
],
140+
"type": "rpm",
141+
"active": False,
142+
"critical_path": False
143+
},
144+
{
145+
"id": 323149,
146+
"global_component": "python",
147+
"name": "f26",
148+
"slas": [
149+
{
150+
"id": 646309,
151+
"sla": "security_fixes",
152+
"eol": "2222-07-01"
153+
},
154+
{
155+
"id": 646303,
156+
"sla": "bug_fixes",
157+
"eol": "2222-07-01"
158+
}
159+
],
160+
"type": "module",
161+
"active": True,
162+
"critical_path": False
163+
}
164+
])
165+
166+
with mock.patch('requests.Session') as mock_requests_session:
167+
mock_rv_found = mock.Mock()
168+
mock_rv_found.status_code = 200
169+
mock_rv_not_found = mock.Mock()
170+
mock_rv_not_found.status_code = 404
171+
mock_session_rv = mock.Mock()
172+
mock_session_rv.head.side_effect = [mock_rv_found, mock_rv_not_found]
173+
mock_requests_session.return_value = mock_session_rv
174+
present, absent = self.handler.audit(pdc)
175+
176+
self.assertEquals(present, set())
177+
self.assertEquals(absent, set())
178+
179+
180+
@mock_pdc
181+
def test_audit_retired_in_pdc_not_cgit(self, pdc):
182+
pdc.add_endpoint('component-branches', 'GET', [
183+
{
184+
"id": 155867,
185+
"global_component": "obexftp",
186+
"name": "f26",
187+
"slas": [
188+
{
189+
"id": 310591,
190+
"sla": "bug_fixes",
191+
"eol": "2017-06-28"
192+
},
193+
{
194+
"id": 310602,
195+
"sla": "security_fixes",
196+
"eol": "2017-06-28"
197+
}
198+
],
199+
"type": "rpm",
200+
"active": False,
201+
"critical_path": False
202+
},
203+
{
204+
"id": 323149,
205+
"global_component": "python",
206+
"name": "f26",
207+
"slas": [
208+
{
209+
"id": 646309,
210+
"sla": "security_fixes",
211+
"eol": "2222-07-01"
212+
},
213+
{
214+
"id": 646303,
215+
"sla": "bug_fixes",
216+
"eol": "2222-07-01"
217+
}
218+
],
219+
"type": "module",
220+
"active": True,
221+
"critical_path": False
222+
}
223+
])
224+
225+
with mock.patch('requests.Session') as mock_requests_session:
226+
mock_rv_not_found = mock.Mock()
227+
mock_rv_not_found.status_code = 404
228+
mock_session_rv = mock.Mock()
229+
mock_session_rv.head.return_value = mock_rv_not_found
230+
mock_requests_session.return_value = mock_session_rv
231+
present, absent = self.handler.audit(pdc)
232+
233+
self.assertEquals(present, {'rpm/obexftp#f26'})
234+
self.assertEquals(absent, set())
235+
236+
@mock_pdc
237+
def test_audit_retired_in_cgit_not_pdc(self, pdc):
238+
pdc.add_endpoint('component-branches', 'GET', [
239+
{
240+
"id": 155867,
241+
"global_component": "obexftp",
242+
"name": "f26",
243+
"slas": [
244+
{
245+
"id": 310591,
246+
"sla": "bug_fixes",
247+
"eol": "2222-06-28"
248+
},
249+
{
250+
"id": 310602,
251+
"sla": "security_fixes",
252+
"eol": "2222-06-28"
253+
}
254+
],
255+
"type": "rpm",
256+
"active": True,
257+
"critical_path": False
258+
},
259+
{
260+
"id": 323149,
261+
"global_component": "python",
262+
"name": "f26",
263+
"slas": [
264+
{
265+
"id": 646309,
266+
"sla": "security_fixes",
267+
"eol": "2222-07-01"
268+
},
269+
{
270+
"id": 646303,
271+
"sla": "bug_fixes",
272+
"eol": "2222-07-01"
273+
}
274+
],
275+
"type": "module",
276+
"active": True,
277+
"critical_path": False
278+
}
279+
])
280+
281+
with mock.patch('requests.Session') as mock_requests_session:
282+
mock_rv_not_found = mock.Mock()
283+
mock_rv_not_found.status_code = 200
284+
mock_session_rv = mock.Mock()
285+
mock_session_rv.head.return_value = mock_rv_not_found
286+
mock_requests_session.return_value = mock_session_rv
287+
present, absent = self.handler.audit(pdc)
288+
289+
self.assertEquals(present, set())
290+
self.assertEquals(absent, {'rpm/obexftp#f26', 'module/python#f26'})
291+
292+
@mock_pdc
293+
def test_initialize(self, pdc):
294+
pdc.add_endpoint('component-branches', 'GET', [
295+
{
296+
"id": 155867,
297+
"global_component": "obexftp",
298+
"name": "f26",
299+
"slas": [
300+
{
301+
"id": 310591,
302+
"sla": "bug_fixes",
303+
"eol": "2017-06-28"
304+
},
305+
{
306+
"id": 310602,
307+
"sla": "security_fixes",
308+
"eol": "2017-06-28"
309+
}
310+
],
311+
"type": "rpm",
312+
"active": False,
313+
"critical_path": False
314+
},
315+
{
316+
"id": 323149,
317+
"global_component": "python",
318+
"name": "f26",
319+
"slas": [
320+
{
321+
"id": 646309,
322+
"sla": "security_fixes",
323+
"eol": "2222-07-01"
324+
},
325+
{
326+
"id": 646303,
327+
"sla": "bug_fixes",
328+
"eol": "2222-07-01"
329+
}
330+
],
331+
"type": "module",
332+
"active": True,
333+
"critical_path": False
334+
}
335+
])
336+
337+
with mock.patch.object(RetireComponentHandler, '_retire_branch') as mock_retire_branch, \
338+
mock.patch.object(RetireComponentHandler, '_is_retired_in_cgit') as mock_is_retired_in_cgit:
339+
mock_is_retired_in_cgit.side_effect = [True, False]
340+
self.handler.initialize(pdc)
341+
self.assertEqual(mock_retire_branch.call_count, 1)

0 commit comments

Comments
 (0)