fix: download directories with http fs

fsspec/implementations/http.py

@@ -1,10 +1,11 @@
 import asyncio
 import io
 import logging
+import os
 import re
 import weakref
 from copy import copy
-from urllib.parse import urlparse
+from urllib.parse import unquote, urlparse
 
 import aiohttp
 import yarl
@@ -156,7 +157,7 @@ async def _ls_real(self, url, detail=True, **kwargs):
         kw.update(kwargs)
         logger.debug(url)
         session = await self.set_session()
-        async with session.get(self.encode_url(url), **self.kwargs) as r:
+        async with session.get(self.encode_url(url), **kw) as r:
             self._raise_not_found_for_status(r, url)
 
             if "Content-Type" in r.headers:
@@ -253,6 +254,9 @@ async def _get_file(
         kw.update(kwargs)
         logger.debug(rpath)
         session = await self.set_session()
+        if await self._isdir(rpath):
+            os.makedirs(unquote(lpath), exist_ok=True)
+            return
         async with session.get(self.encode_url(rpath), **kw) as r:
             try:
                 size = int(r.headers["content-length"])
@@ -264,7 +268,7 @@ async def _get_file(
             if isfilelike(lpath):
                 outfile = lpath
             else:
-                outfile = open(lpath, "wb")  # noqa: ASYNC230
+                outfile = open(unquote(lpath), "wb")  # noqa: ASYNC230
 
             try:
                 chunk = True

fsspec/implementations/tests/test_http.py

@@ -319,6 +319,16 @@ def test_download(server, tmpdir):
     assert open(fn, "rb").read() == data
 
 
+def test_download_dir(server, tmpdir):
+    h = fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true "})
+    url = server.address + "/index/"
+    fn = os.path.join(tmpdir, "adir")
+    h.get(url, fn, recursive=True)
+    assert os.path.exists(fn)
+    assert os.path.exists(os.path.join(fn, "realfile"))
+    assert open(os.path.join(fn, "realfile"), "rb").read() == data
+
+
 def test_multi_download(server, tmpdir):
     h = fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true "})
     urla = server.realfile