Standardize requests made by DOIDownloaders (#514)

Respect user's decisions when defining the `DOIDownloader` with respect
to arguments passed to `requests.get` whenever we call that function.
This way, all calls made by `DOIDownloaders` and the repository classes
make use of the same arguments, including `timeout`, `headers`, etc.

Author: santisoler

pooch/core.py

@@ -707,7 +707,12 @@ def load_registry_from_doi(self) -> None:
 
         # Create a repository instance
         doi = self.base_url.replace("doi:", "")
-        repository = doi_to_repository(doi)
+        repository = doi_to_repository(
+            doi,
+            headers=downloader.headers,
+            timeout=downloader.timeout,
+            **downloader.kwargs,
+        )
 
         # Call registry population for this repository
         return repository.populate_registry(self)

pooch/downloaders.py

@@ -603,9 +603,17 @@ class DOIDownloader:  # pylint: disable=too-few-public-methods
 
     """
 
-    def __init__(self, progressbar=False, chunk_size=1024, headers=None, **kwargs):
+    def __init__(
+        self,
+        progressbar=False,
+        chunk_size=1024,
+        headers=None,
+        timeout=DEFAULT_TIMEOUT,
+        **kwargs,
+    ):
         self.kwargs = kwargs
         self.headers = headers if headers is not None else REQUESTS_HEADERS
+        self.timeout = timeout
         self.progressbar = progressbar
         self.chunk_size = chunk_size
 
@@ -630,7 +638,13 @@ def __call__(self, url, output_file, pooch):
         """
 
         parsed_url = parse_url(url)
-        data_repository = doi_to_repository(parsed_url["netloc"])
+
+        data_repository = doi_to_repository(
+            parsed_url["netloc"],
+            headers=self.headers,
+            timeout=self.timeout,
+            **self.kwargs,
+        )
 
         # Resolve the URL
         file_name = parsed_url["path"]
@@ -644,19 +658,22 @@ def __call__(self, url, output_file, pooch):
             progressbar=self.progressbar,
             chunk_size=self.chunk_size,
             headers=self.headers,
+            timeout=self.timeout,
             **self.kwargs,
         )
         downloader(download_url, output_file, pooch)
 
 
-def doi_to_url(doi):
+def doi_to_url(doi, **kwargs):
     """
     Follow a DOI link to resolve the URL of the archive.
 
     Parameters
     ----------
     doi : str
         The DOI of the archive.
+    **kwargs
+        All keyword arguments will be passed to :func:`requests.get`.
 
     Returns
     -------
@@ -669,15 +686,16 @@ def doi_to_url(doi):
 
     # Use doi.org to resolve the DOI to the repository website.
     response = requests.get(
-        f"https://doi.org/{doi}", headers=REQUESTS_HEADERS, timeout=DEFAULT_TIMEOUT
+        f"https://doi.org/{doi}",
+        **kwargs,
     )
     url = response.url
     if 400 <= response.status_code < 600:
         response.raise_for_status()
     return url
 
 
-def doi_to_repository(doi):
+def doi_to_repository(doi, **kwargs):
     """
     Instantiate a data repository instance from a given DOI.
 
@@ -688,6 +706,10 @@ def doi_to_repository(doi):
     ----------
     doi : str
         The DOI of the archive.
+    **kwargs
+        All keyword arguments will be passed also as ``**kwargs`` to the
+        :meth:`DataRepository.initialize` method, that will ultimately get
+        passed to :func:`requests.get`.
 
     Returns
     -------
@@ -708,7 +730,7 @@ def doi_to_repository(doi):
     ]
 
     # Extract the DOI and the repository information
-    archive_url = doi_to_url(doi)
+    archive_url = doi_to_url(doi, **kwargs)
 
     # Try the converters one by one until one of them returned a URL
     data_repository = None
@@ -717,6 +739,7 @@ def doi_to_repository(doi):
             data_repository = repo.initialize(
                 archive_url=archive_url,
                 doi=doi,
+                **kwargs,
             )
 
     if data_repository is None:
@@ -732,7 +755,7 @@ def doi_to_repository(doi):
 
 class DataRepository:  # pylint: disable=too-few-public-methods, missing-class-docstring
     @classmethod
-    def initialize(cls, doi, archive_url):  # pylint: disable=unused-argument
+    def initialize(cls, doi, archive_url, **kwargs):  # pylint: disable=unused-argument
         """
         Initialize the data repository if the given URL points to a
         corresponding repository.
@@ -786,14 +809,15 @@ def populate_registry(self, pooch):
 class ZenodoRepository(DataRepository):  # pylint: disable=missing-class-docstring
     base_api_url = "https://zenodo.org/api/records"
 
-    def __init__(self, doi, archive_url):
+    def __init__(self, doi, archive_url, **kwargs):
         self.archive_url = archive_url
         self.doi = doi
         self._api_response = None
         self._api_version = None
+        self.kwargs = kwargs
 
     @classmethod
-    def initialize(cls, doi, archive_url):
+    def initialize(cls, doi, archive_url, **kwargs):
         """
         Initialize the data repository if the given URL points to a
         corresponding repository.
@@ -809,14 +833,17 @@ def initialize(cls, doi, archive_url):
             The DOI that identifies the repository
         archive_url : str
             The resolved URL for the DOI
+        **kwargs
+            All keyword arguments given when creating an instance of this class
+            will be passed to :func:`requests.get`.
         """
 
         # Check whether this is a Zenodo URL
         parsed_archive_url = parse_url(archive_url)
         if parsed_archive_url["netloc"] != "zenodo.org":
             return None
 
-        return cls(doi, archive_url)
+        return cls(doi, archive_url, **kwargs)
 
     @property
     def api_response(self):
@@ -827,9 +854,7 @@ def api_response(self):
 
             article_id = self.archive_url.split("/")[-1]
             self._api_response = requests.get(
-                f"{self.base_api_url}/{article_id}",
-                headers=REQUESTS_HEADERS,
-                timeout=DEFAULT_TIMEOUT,
+                f"{self.base_api_url}/{article_id}", **self.kwargs
             ).json()
 
         return self._api_response
@@ -938,13 +963,14 @@ def populate_registry(self, pooch):
 
 
 class FigshareRepository(DataRepository):  # pylint: disable=missing-class-docstring
-    def __init__(self, doi, archive_url):
+    def __init__(self, doi, archive_url, **kwargs):
         self.archive_url = archive_url
         self.doi = doi
         self._api_response = None
+        self.kwargs = kwargs
 
     @classmethod
-    def initialize(cls, doi, archive_url):
+    def initialize(cls, doi, archive_url, **kwargs):
         """
         Initialize the data repository if the given URL points to a
         corresponding repository.
@@ -960,14 +986,17 @@ def initialize(cls, doi, archive_url):
             The DOI that identifies the repository
         archive_url : str
             The resolved URL for the DOI
+        **kwargs
+            All keyword arguments given when creating an instance of this class
+            will be passed to :func:`requests.get`.
         """
 
         # Check whether this is a Figshare URL
         parsed_archive_url = parse_url(archive_url)
         if parsed_archive_url["netloc"] != "figshare.com":
             return None
 
-        return cls(doi, archive_url)
+        return cls(doi, archive_url, **kwargs)
 
     def _parse_version_from_doi(self):
         """
@@ -995,8 +1024,7 @@ def api_response(self):
             # Use the figshare API to find the article ID from the DOI
             article = requests.get(
                 f"https://api.figshare.com/v2/articles?doi={self.doi}",
-                headers=REQUESTS_HEADERS,
-                timeout=DEFAULT_TIMEOUT,
+                **self.kwargs,
             ).json()[0]
             article_id = article["id"]
             # Parse desired version from the doi
@@ -1023,9 +1051,7 @@ def api_response(self):
                     f"{article_id}/versions/{version}"
                 )
             # Make the request and return the files in the figshare repository
-            response = requests.get(
-                api_url, headers=REQUESTS_HEADERS, timeout=DEFAULT_TIMEOUT
-            )
+            response = requests.get(api_url, **self.kwargs)
             response.raise_for_status()
             self._api_response = response.json()["files"]
 
@@ -1069,13 +1095,14 @@ def populate_registry(self, pooch):
 
 
 class DataverseRepository(DataRepository):  # pylint: disable=missing-class-docstring
-    def __init__(self, doi, archive_url):
+    def __init__(self, doi, archive_url, **kwargs):
         self.archive_url = archive_url
         self.doi = doi
         self._api_response = None
+        self.kwargs = kwargs
 
     @classmethod
-    def initialize(cls, doi, archive_url):
+    def initialize(cls, doi, archive_url, **kwargs):
         """
         Initialize the data repository if the given URL points to a
         corresponding repository.
@@ -1091,21 +1118,24 @@ def initialize(cls, doi, archive_url):
             The DOI that identifies the repository
         archive_url : str
             The resolved URL for the DOI
+        **kwargs
+            All keyword arguments given when creating an instance of this class
+            will be passed to :func:`requests.get`.
         """
         # Access the DOI as if this was a DataVerse instance
-        response = cls._get_api_response(doi, archive_url)
+        response = cls._get_api_response(doi, archive_url, **kwargs)
 
         # If we failed, this is probably not a DataVerse instance
         if 400 <= response.status_code < 600:
             return None
 
         # Initialize the repository and overwrite the api response
-        repository = cls(doi, archive_url)
+        repository = cls(doi, archive_url, **kwargs)
         repository.api_response = response
         return repository
 
     @classmethod
-    def _get_api_response(cls, doi, archive_url):
+    def _get_api_response(cls, doi, archive_url, **kwargs):
         """
         Perform the actual API request
 
@@ -1119,8 +1149,7 @@ def _get_api_response(cls, doi, archive_url):
         response = requests.get(
             f"{parsed['protocol']}://{parsed['netloc']}/api/datasets/"
             f":persistentId?persistentId=doi:{doi}",
-            headers=REQUESTS_HEADERS,
-            timeout=DEFAULT_TIMEOUT,
+            **kwargs,
         )
         return response
 
@@ -1130,7 +1159,7 @@ def api_response(self):
 
         if self._api_response is None:
             self._api_response = self._get_api_response(
-                self.doi, self.archive_url
+                self.doi, self.archive_url, **self.kwargs
             )  # pragma: no cover
 
         return self._api_response