[CI] Metrics upload crashing occasionally #127273
Description
The metrics job seems to be crashing occasionally (about five times since it was deployed about a week or so ago). The traceback from the latest crash:
Traceback (most recent call last):
File "/usr/local/lib/python3.12/site-packages/urllib3/response.py", line 748, in _error_catcher
yield
File "/usr/local/lib/python3.12/site-packages/urllib3/response.py", line 1209, in read_chunked
chunk = self._handle_chunk(amt)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/urllib3/response.py", line 1146, in _handle_chunk
value = self._fp._safe_read(amt) # type: ignore[union-attr]
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/http/client.py", line 642, in _safe_read
raise IncompleteRead(data, amt-len(data))
http.client.IncompleteRead: IncompleteRead(3634 bytes read, 6606 more expected)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.12/site-packages/requests/models.py", line 820, in generate
yield from self.raw.stream(chunk_size, decode_content=True)
File "/usr/local/lib/python3.12/site-packages/urllib3/response.py", line 1057, in stream
yield from self.read_chunked(amt, decode_content=decode_content)
File "/usr/local/lib/python3.12/site-packages/urllib3/response.py", line 1189, in read_chunked
with self._error_catcher():
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/contextlib.py", line 158, in __exit__
self.gen.throw(value)
File "/usr/local/lib/python3.12/site-packages/urllib3/response.py", line 775, in _error_catcher
raise ProtocolError(f"Connection broken: {e!r}", e) from e
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(3634 bytes read, 6606 more expected)', IncompleteRead(3634 bytes read, 6606 more expected))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "//metrics.py", line 184, in <module>
main()
File "//metrics.py", line 169, in main
current_metrics = get_metrics(github_repo, workflows_to_track)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "//metrics.py", line 53, in get_metrics
workflow_run = next(workflow_runs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/PaginatedList.py", line 84, in __iter__
newElements = self._grow()
^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/PaginatedList.py", line 95, in _grow
newElements = self._fetchNextPage()
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/PaginatedList.py", line 320, in _fetchNextPage
headers, data = self.__requester.requestJsonAndCheck(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/Requester.py", line 586, in requestJsonAndCheck
return self.__check(*self.requestJson(verb, url, parameters, headers, input, self.__customConnection(url)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/Requester.py", line 856, in requestJson
return self.__requestEncode(cnx, verb, url, parameters, headers, input, encode)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/Requester.py", line 977, in __requestEncode
status, responseHeaders, output = self.__requestRaw(cnx, verb, url, requestHeaders, encoded_input)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/Requester.py", line 1011, in __requestRaw
response = cnx.getresponse()
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/github/Requester.py", line 202, in getresponse
r = verb(
^^^^^
File "/usr/local/lib/python3.12/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/requests/sessions.py", line 746, in send
r.content
File "/usr/local/lib/python3.12/site-packages/requests/models.py", line 902, in content
self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b""
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/requests/models.py", line 822, in generate
raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(3634 bytes read, 6606 more expected)', IncompleteRead(3634 bytes read, 6606 more expected))
Putting this as low priority as k8s will automatically restart the pod if it crashes, so this is barely noticeable.
XRef the internal bug: b/381693531