make github _open() return HTTPFile when getting content from download_url

Author: Thomas Gilgenast

fsspec/implementations/github.py

@@ -2,8 +2,10 @@
 
 import requests
 
+from ..asyn import get_loop, sync
 from ..spec import AbstractFileSystem
 from ..utils import infer_storage_options
+from .http import HTTPFile, HTTPFileSystem
 from .memory import MemoryFile
 
 # TODO: add GIST backend, would be very similar
@@ -64,6 +66,11 @@ def __init__(
         self.root = sha
         self.ls("")
 
+        # prepare elements needed to return HTTPFile
+        self.http_fs = HTTPFileSystem(**kwargs)
+        self.loop = get_loop()
+        self.session = sync(self.loop, self.http_fs.set_session)
+
     @property
     def kw(self):
         if self.username:
@@ -245,9 +252,16 @@ def _open(
 
         # we land here if the content was not present in the first response
         # (regular file over 1MB or git-lfs tracked file)
-        # in this case, we get the content from the download_url
-        r = requests.get(content_json["download_url"], timeout=self.timeout, **self.kw)
-        if r.status_code == 404:
-            raise FileNotFoundError(path)
-        r.raise_for_status()
-        return MemoryFile(None, None, r.content)
+        # in this case, we get return an HTTPFile object wrapping the
+        # download_url
+        return HTTPFile(
+            self.http_fs,
+            content_json["download_url"],
+            session=self.session,
+            block_size=block_size,
+            autocommit=autocommit,
+            cache_options=cache_options,
+            size=content_json["size"],
+            loop=self.loop,
+            **kwargs,
+        )

fsspec/implementations/tests/test_github.py

@@ -9,8 +9,14 @@ def test_github_open_small_file():
 
 def test_github_open_large_file():
     # test opening a large file >1 MB
-    with fsspec.open("github://mwaskom:seaborn-data@83bfba7/brain_networks.csv") as f:
-        assert f.readline().startswith(b"network,1,1,2,2")
+    # use block_size=0 to get a streaming interface to the file, ensuring that
+    # we fetch only the parts we need instead of downloading the full file all
+    # at once
+    with fsspec.open(
+        "github://mwaskom:seaborn-data@83bfba7/brain_networks.csv", block_size=0
+    ) as f:
+        # read only the first 20 bytes of the file
+        assert f.read(20).startswith(b"network,1,1,2,2,3,3,")
 
 
 def test_github_open_lfs_file():

pyproject.toml

@@ -56,7 +56,7 @@ full = [
 fuse = ["fusepy"]
 gcs = ["gcsfs"]
 git = ["pygit2"]
-github = ["requests"]
+github = ["fsspec[http]", "requests"]
 gs = ["gcsfs"]
 gui = ["panel"]
 hdfs = ["pyarrow >= 1"]