fixes #335

Author: jph00

fastcore/_nbdev.py

@@ -200,6 +200,7 @@
          "urljson": "03b_net.ipynb",
          "urlcheck": "03b_net.ipynb",
          "urlclean": "03b_net.ipynb",
+         "urlretrieve": "03b_net.ipynb",
          "urlsave": "03b_net.ipynb",
          "urlvalid": "03b_net.ipynb",
          "urlrequest": "03b_net.ipynb",

fastcore/net.py

@@ -10,8 +10,8 @@
            'HTTP422UnprocessableEntityError', 'HTTP423LockedError', 'HTTP424FailedDependencyError',
            'HTTP425TooEarlyError', 'HTTP426UpgradeRequiredError', 'HTTP428PreconditionRequiredError',
            'HTTP429TooManyRequestsError', 'HTTP431HeaderFieldsTooLargeError', 'HTTP451LegalReasonsError', 'urlopen',
-           'urlread', 'urljson', 'urlcheck', 'urlclean', 'urlsave', 'urlvalid', 'urlrequest', 'urlsend', 'do_request',
-           'start_server', 'start_client']
+           'urlread', 'urljson', 'urlcheck', 'urlclean', 'urlretrieve', 'urlsave', 'urlvalid', 'urlrequest', 'urlsend',
+           'do_request', 'start_server', 'start_client']
 
 # Cell
 from .imports import *
@@ -21,10 +21,10 @@
 from .parallel import *
 from functools import wraps
 
-import json,urllib
+import json,urllib,contextlib
 import socket,urllib.request,http,urllib
 from contextlib import contextmanager,ExitStack
-from urllib.request import Request
+from urllib.request import Request,urlretrieve,install_opener
 from urllib.error import HTTPError,URLError
 from urllib.parse import urlencode,urlparse,urlunparse
 from http.client import InvalidURL
@@ -75,6 +75,7 @@ class HTTP5xxServerError(HTTPError):
 # Cell
 _opener = urllib.request.build_opener()
 _opener.addheaders = list(url_default_headers.items())
+install_opener(_opener)
 
 _httperrors = (
     (400,'Bad Request'),(401,'Unauthorized'),(402,'Payment Required'),(403,'Forbidden'),(404,'Not Found'),
@@ -137,15 +138,41 @@ def urlclean(url):
     return urlunparse(urlparse(str(url))[:3]+('','',''))
 
 # Cell
-def urlsave(url, dest=None):
+def urlretrieve(url, filename=None, reporthook=None, data=None):
+    "Same as `urllib.request.urlretrieve` but also works with `Request` objects"
+    with contextlib.closing(urlopen(url, data)) as fp:
+        headers = fp.info()
+        if filename: tfp = open(filename, 'wb')
+        else:
+            tfp = tempfile.NamedTemporaryFile(delete=False)
+            filename = tfp.name
+
+        with tfp:
+            bs,size,read,blocknum = 1024*8,-1,0,0
+            if "content-length" in headers: size = int(headers["Content-Length"])
+            if reporthook: reporthook(blocknum, bs, size)
+            while True:
+                block = fp.read(bs)
+                if not block: break
+                read += len(block)
+                tfp.write(block)
+                blocknum += 1
+                if reporthook: reporthook(blocknum, bs, size)
+
+    if size >= 0 and read < size:
+        raise ContentTooShortError(f"retrieval incomplete: got only {read} out of {size} bytes", headers)
+    return filename,headers
+
+# Cell
+def urlsave(url, dest=None, reporthook=None):
     "Retrieve `url` and save based on its name"
-    res = urlread(urlwrap(url), decode=False)
     name = urlclean(Path(url).name)
     if dest is None: dest = name
     dest = Path(dest)
     if dest.is_dir(): dest = dest/name
-    Path(dest).write_bytes(res)
-    return dest
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    nm,msg = urlretrieve(url, dest, reporthook)
+    return nm
 
 # Cell
 def urlvalid(x):

nbs/03b_net.ipynb

@@ -23,10 +23,10 @@
     "from fastcore.parallel import *\n",
     "from functools import wraps\n",
     "\n",
-    "import json,urllib\n",
+    "import json,urllib,contextlib\n",
     "import socket,urllib.request,http,urllib\n",
     "from contextlib import contextmanager,ExitStack\n",
-    "from urllib.request import Request\n",
+    "from urllib.request import Request,urlretrieve,install_opener\n",
     "from urllib.error import HTTPError,URLError\n",
     "from urllib.parse import urlencode,urlparse,urlunparse\n",
     "from http.client import InvalidURL"
@@ -238,6 +238,7 @@
     "#export\n",
     "_opener = urllib.request.build_opener()\n",
     "_opener.addheaders = list(url_default_headers.items())\n",
+    "install_opener(_opener)\n",
     "\n",
     "_httperrors = (\n",
     "    (400,'Bad Request'),(401,'Unauthorized'),(402,'Payment Required'),(403,'Forbidden'),(404,'Not Found'),\n",
@@ -367,30 +368,55 @@
    "outputs": [],
    "source": [
     "#export\n",
-    "def urlsave(url, dest=None):\n",
+    "def urlretrieve(url, filename=None, reporthook=None, data=None):\n",
+    "    \"Same as `urllib.request.urlretrieve` but also works with `Request` objects\"\n",
+    "    with contextlib.closing(urlopen(url, data)) as fp:\n",
+    "        headers = fp.info()\n",
+    "        if filename: tfp = open(filename, 'wb')\n",
+    "        else:\n",
+    "            tfp = tempfile.NamedTemporaryFile(delete=False)\n",
+    "            filename = tfp.name\n",
+    "\n",
+    "        with tfp:\n",
+    "            bs,size,read,blocknum = 1024*8,-1,0,0\n",
+    "            if \"content-length\" in headers: size = int(headers[\"Content-Length\"])\n",
+    "            if reporthook: reporthook(blocknum, bs, size)\n",
+    "            while True:\n",
+    "                block = fp.read(bs)\n",
+    "                if not block: break\n",
+    "                read += len(block)\n",
+    "                tfp.write(block)\n",
+    "                blocknum += 1\n",
+    "                if reporthook: reporthook(blocknum, bs, size)\n",
+    "\n",
+    "    if size >= 0 and read < size:\n",
+    "        raise ContentTooShortError(f\"retrieval incomplete: got only {read} out of {size} bytes\", headers)\n",
+    "    return filename,headers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "def urlsave(url, dest=None, reporthook=None):\n",
     "    \"Retrieve `url` and save based on its name\"\n",
-    "    res = urlread(urlwrap(url), decode=False)\n",
     "    name = urlclean(Path(url).name)\n",
     "    if dest is None: dest = name\n",
     "    dest = Path(dest)\n",
     "    if dest.is_dir(): dest = dest/name\n",
-    "    Path(dest).write_bytes(res)\n",
-    "    return dest"
+    "    dest.parent.mkdir(parents=True, exist_ok=True)\n",
+    "    nm,msg = urlretrieve(url, dest, reporthook)\n",
+    "    return nm"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[Path('/tmp/tmpr3kv5gq_/index.html')]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "#skip\n",
     "with tempfile.TemporaryDirectory() as d: urlsave('http://www.google.com/index.html', d)"
@@ -646,7 +672,6 @@
       "Converted 05_transform.ipynb.\n",
       "Converted 07_meta.ipynb.\n",
       "Converted 08_script.ipynb.\n",
-      "Converted index.ipynb.\n",
       "Converted parallel_win.ipynb.\n"
      ]
     }