@@ -684,15 +684,20 @@ def _get_url_response(self, full_url, retries=5):
684684 retry_after = ex.headers.get("RateLimit-Reset")
685685 if retry_after:
686686 wait_time = int(retry_after)
687- else:
688- raise BuildkiteException(
689- "Failed to open {}: {} - {}".format(url, ex.code, ex.reason)
690- )
687+ time.sleep(wait_time + 1)
688+ continue # Retry after waiting
689+ else:
690+ raise BuildkiteException(
691+ "Failed to open {}: {} - {}".format(full_url, ex.code, ex.reason)
692+ )
693+ except Exception as ex:
694+ # Retry on other exceptions
695+ pass
691696
692- if not success :
693- raise BuildkiteException(f"Failed to open {url} after {retries} retries.")
697+ if attempt < retries - 1 :
698+ time.sleep(2 ** attempt) # Exponential backoff
694699
695- return all_items
700+ raise BuildkiteException(f"Failed to open {full_url} after {retries} retries.")
696701
697702 def _get_next_page_url(self, headers):
698703 """Parses the headers to determine if there are more pagination pages."""
@@ -702,28 +707,18 @@ def _get_next_page_url(self, headers):
702707 match = self._NEXT_PAGE_PATTERN.search(link_header)
703708 return match.group('url') if match else None
704709
705- time.sleep(wait_time)
706- else:
707- raise BuildkiteException(
708- "Failed to open {}: {} - {}".format(full_url, ex.code, ex.reason)
709- )
710710 def _build_url_with_params(self, url, params=[]):
711711 """Builds a URL with the given query parameters."""
712712 params_str = "".join("&{}={}".format(k, v) for k, v in params)
713713 return "{}?access_token={}{}".format(url, self._token, params_str)
714714
715- raise BuildkiteException(f"Failed to open {url} after {retries} retries.")
716715 def _fetch_data_as_text(self, url, params=[], retries=5) -> str:
717716 """Returns the decode utf-8 representation of the _get_url_response."""
718- return self._get_url_response(url, params, retries).read().decode("utf-8", "ignore")
719717 url = self._build_url_with_params(url, params)
720718 return self._get_url_response(url, retries)[0]
721719
722- def _open_url_with_paganation(self, url, params=[], retries=5) -> List:
723720 def _fetch_all_pages_as_json(self, url, params=[], retries=5) -> List:
724721 """Fetch all items iteratively across all pages."""
725- params = params + [("per_page", "100")]
726- params_str = "".join("&{}={}".format(k, v) for k, v in params)
727722 next_url = self._build_url_with_params(url, params + [("per_page", "100")])
728723
729724 all_items = []
0 commit comments