Hide exposed import to requests

Closes #144

Author: cthoyt

pyproject.toml

@@ -77,7 +77,7 @@ xml = ["lxml"]
 pandas = ["pandas"]
 aws = ["boto3"]
 ratelimit = ["ratelimit", "requests"]
-bs4 = ["bs4"]
+bs4 = ["bs4", "requests"]
 yaml = ["pyyaml"]
 pydantic = ["pydantic"]
 cli = ["click"]

src/pystow/utils/__init__.py

@@ -32,7 +32,6 @@
 from urllib.parse import urlparse
 from uuid import uuid4
 
-import requests
 from tqdm.auto import tqdm
 
 from .download import (
@@ -88,7 +87,14 @@
     stream_write_pydantic_jsonl,
     write_pydantic_jsonl,
 )
-from .safe_open import is_url, open_inner_zipfile, open_url, safe_open, safe_open_dict_reader
+from .safe_open import (
+    is_url,
+    open_inner_zipfile,
+    open_url,
+    safe_open,
+    safe_open_dict_reader,
+    safe_open_json,
+)
 from ..constants import README_TEXT, TimeoutHint
 
 if TYPE_CHECKING:
@@ -168,6 +174,7 @@
     "safe_open",
     "safe_open_dict_reader",
     "safe_open_dict_writer",
+    "safe_open_json",
     "safe_open_reader",
     "safe_open_writer",
     "safe_tarfile_open",
@@ -707,8 +714,8 @@ def get_commit(org: str, repo: str, provider: str = "git") -> str:
         lines = (line.strip().split("\t") for line in output.decode("utf8").splitlines())
         rv = next(line[0] for line in lines if line[1] == "HEAD")
     elif provider == "github":
-        res = requests.get(f"https://api.github.com/repos/{org}/{repo}/branches/master", timeout=15)
-        res_json = res.json()
+        url = f"https://api.github.com/repos/{org}/{repo}/branches/master"
+        res_json = safe_open_json(url)
         rv = res_json["commit"]["sha"]
     else:
         raise ValueError(f"invalid implementation: {provider}")
@@ -855,6 +862,7 @@ def get_soup(
 
     :returns: A BeautifulSoup object
     """
+    import requests
     from bs4 import BeautifulSoup
 
     headers = {}

src/pystow/utils/safe_open.py

@@ -6,6 +6,7 @@
 import csv
 import gzip
 import io
+import json
 import typing
 import urllib.request
 import zipfile
@@ -32,6 +33,7 @@
     "open_url",
     "safe_open",
     "safe_open_dict_reader",
+    "safe_open_json",
 ]
 
 
@@ -139,6 +141,12 @@ def safe_open(  # noqa:C901
         raise TypeError(f"unsupported type for opening: {type(path)} - {path}")
 
 
+def safe_open_json(path_or_url: str) -> Any:
+    """Safely open a file and parse as JSON."""
+    with safe_open(path_or_url, representation="text", operation="read") as file:
+        return json.load(file)
+
+
 # docstr-coverage:excused `overload`
 @typing.overload
 @contextlib.contextmanager

tests/test_utils/test_utils.py

@@ -49,6 +49,7 @@
     read_zipfile_xml,
     safe_open,
     safe_open_dict_reader,
+    safe_open_json,
     safe_open_reader,
     safe_open_writer,
     tarfile_writestr,
@@ -517,6 +518,13 @@ def test_open_url(self) -> None:
         ) as file:
             self.assertIn("sleep 5", file.read().decode("utf-8"))
 
+    def test_get_json(self) -> None:
+        """Test getting JSON."""
+        url = "https://zenodo.org/records/15826754/files/configuration.json?download=1"
+        x = safe_open_json(url)
+        self.assertIsInstance(x, dict)
+        self.assertIn("name", x)
+
 
 class TestDownload(unittest.TestCase):
     """Tests for downloading."""