fix: add retrys for zenodo requests with tenacity (#1861)

Author: lkstrp

doc/release_notes.rst

@@ -9,6 +9,9 @@ Release Notes
 Upcoming Release
 ================
 
+* Added automatic retry for some (Zenodo) HTTP requests to handle transient errors 
+  like rate limiting and server errors.
+
 * Fixed `ValueError` in `prepare_sector_network.py` in function `add_storage_and_grids`
   when running with few nodes such that they are all already connected by existing gas
   lines. (https://github.com/PyPSA/pypsa-eur/pull/1780)

envs/environment.yaml

@@ -46,6 +46,7 @@ dependencies:
 - country_converter
 - geopy
 - tqdm
+- tenacity
 - pytz
 - jpype1
 - pyxlsb

rules/common.smk

@@ -6,6 +6,13 @@ import copy
 from functools import partial, lru_cache
 
 import os, sys, glob
+import requests
+from tenacity import (
+    retry as tenacity_retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+)
 
 path = workflow.source_path("../scripts/_helpers.py")
 sys.path.insert(0, os.path.dirname(path))
@@ -113,6 +120,13 @@ def input_custom_extra_functionality(w):
     return []
 
 
+@tenacity_retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    retry=retry_if_exception_type(
+        (requests.HTTPError, requests.ConnectionError, requests.Timeout)
+    ),
+)
 def has_internet_access(url: str = "https://www.zenodo.org", timeout: int = 5) -> bool:
     """
     Checks if internet connection is available by sending a HEAD request
@@ -125,14 +139,14 @@ def has_internet_access(url: str = "https://www.zenodo.org", timeout: int = 5) -
     Returns:
     - bool: True if the internet is available, otherwise False.
     """
-    try:
-        # Send a HEAD request to avoid fetching full response
-        response = requests.head(url, timeout=timeout, allow_redirects=True)
-        return response.status_code == 200
-    except requests.ConnectionError:  # (e.g., no internet, DNS issues)
-        return False
-    except requests.Timeout:  # (e.g., slow or no network)
-        return False
+    # Send a HEAD request to avoid fetching full response
+    response = requests.head(url, timeout=timeout, allow_redirects=True)
+    # Raise HTTPError for transient errors
+    # 429: Too Many Requests (rate limiting)
+    # 500, 502, 503, 504: Server errors
+    if response.status_code in (429, 500, 502, 503, 504):
+        response.raise_for_status()
+    return response.status_code == 200
 
 
 def solved_previous_horizon(w):

scripts/_helpers.py

@@ -23,6 +23,14 @@
 import xarray as xr
 import yaml
 from snakemake.utils import update_config
+from tenacity import (
+    retry as tenacity_retry,
+)
+from tenacity import (
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
 from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
@@ -407,17 +415,31 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False):
     return costs
 
 
+@tenacity_retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    retry=retry_if_exception_type(
+        (requests.HTTPError, requests.ConnectionError, requests.Timeout)
+    ),
+)
 def progress_retrieve(url, file, disable=False):
     headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
     # Hotfix - Bug, tqdm not working with disable=False
     disable = True
 
+    # Raise HTTPError for transient errors
+    # 429: Too Many Requests (rate limiting)
+    # 500, 502, 503, 504: Server errors
     if disable:
         response = requests.get(url, headers=headers, stream=True)
+        if response.status_code in (429, 500, 502, 503, 504):
+            response.raise_for_status()
         with open(file, "wb") as f:
             f.write(response.content)
     else:
         response = requests.get(url, headers=headers, stream=True)
+        if response.status_code in (429, 500, 502, 503, 504):
+            response.raise_for_status()
         total_size = int(response.headers.get("content-length", 0))
         chunk_size = 1024
 
@@ -834,12 +856,24 @@ def update_config_from_wildcards(config, w, inplace=True):
         return config
 
 
+@tenacity_retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    retry=retry_if_exception_type(
+        (requests.HTTPError, requests.ConnectionError, requests.Timeout)
+    ),
+)
 def get_checksum_from_zenodo(file_url):
     parts = file_url.split("/")
     record_id = parts[parts.index("records") + 1]
     filename = parts[-1]
 
     response = requests.get(f"https://zenodo.org/api/records/{record_id}", timeout=30)
+    # Raise HTTPError for transient errors
+    # 429: Too Many Requests (rate limiting)
+    # 500, 502, 503, 504: Server errors
+    if response.status_code in (429, 500, 502, 503, 504):
+        response.raise_for_status()
     response.raise_for_status()
     data = response.json()