Merge pull request #480 from fastai/print-body

add body to error message

Author: jph00

fastcore/net.py

@@ -17,12 +17,12 @@
 # %% ../nbs/03b_net.ipynb 1
 from .utils import *
 from .parallel import *
-from functools import wraps
 
+from functools import wraps
 import json,urllib,contextlib
 import socket,urllib.request,http,urllib
 from contextlib import contextmanager,ExitStack
-from urllib.request import Request,urlretrieve,install_opener
+from urllib.request import Request,urlretrieve,install_opener,HTTPErrorProcessor
 from urllib.error import HTTPError,URLError
 from urllib.parse import urlencode,urlparse,urlunparse
 from http.client import InvalidURL
@@ -104,28 +104,31 @@ def urlopen(url, data=None, headers=None, timeout=None, **kwargs):
     if data is not None:
         if not isinstance(data, (str,bytes)): data = urlencode(data)
         if not isinstance(data, bytes): data = data.encode('ascii')
-    return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)
+    try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)
+    except HTTPError as e: 
+        e.msg += f"\n====Error Body====\n{e.read().decode(errors='ignore')}"
+        raise
 
-# %% ../nbs/03b_net.ipynb 18
+# %% ../nbs/03b_net.ipynb 20
 def urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs):
     "Retrieve `url`, using `data` dict or `kwargs` to `POST` if present"
     try:
         with urlopen(url, data=data, headers=headers, timeout=timeout, **kwargs) as u: res,hdrs = u.read(),u.headers
     except HTTPError as e:
-        if 400 <= e.code < 500: raise ExceptionsHTTP[e.code](e.url, e.hdrs, e.fp) from None
+        if 400 <= e.code < 500: raise ExceptionsHTTP[e.code](e.url, e.hdrs, e.fp, msg=e.msg) from None
         else: raise
 
     if decode: res = res.decode()
     if return_json: res = loads(res)
     return (res,dict(hdrs)) if return_headers else res
 
-# %% ../nbs/03b_net.ipynb 19
+# %% ../nbs/03b_net.ipynb 21
 def urljson(url, data=None, timeout=None):
     "Retrieve `url` and decode json"
     res = urlread(url, data=data, timeout=timeout)
     return json.loads(res) if res else {}
 
-# %% ../nbs/03b_net.ipynb 21
+# %% ../nbs/03b_net.ipynb 23
 def urlcheck(url, timeout=10):
     if not url: return True
     try:
@@ -134,12 +137,12 @@ def urlcheck(url, timeout=10):
     except socket.timeout: return False
     except InvalidURL: return False
 
-# %% ../nbs/03b_net.ipynb 22
+# %% ../nbs/03b_net.ipynb 24
 def urlclean(url):
     "Remove fragment, params, and querystring from `url` if present"
     return urlunparse(urlparse(str(url))[:3]+('','',''))
 
-# %% ../nbs/03b_net.ipynb 24
+# %% ../nbs/03b_net.ipynb 26
 def urlretrieve(url, filename=None, reporthook=None, data=None, timeout=None):
     "Same as `urllib.request.urlretrieve` but also works with `Request` objects"
     with contextlib.closing(urlopen(url, data, timeout=timeout)) as fp:
@@ -165,43 +168,43 @@ def urlretrieve(url, filename=None, reporthook=None, data=None, timeout=None):
         raise ContentTooShortError(f"retrieval incomplete: got only {read} out of {size} bytes", headers)
     return filename,headers
 
-# %% ../nbs/03b_net.ipynb 25
+# %% ../nbs/03b_net.ipynb 27
 def urldest(url, dest=None):
     name = urlclean(Path(url).name)
     if dest is None: dest = name
     dest = Path(dest)
     return dest/name if dest.is_dir() else dest
 
-# %% ../nbs/03b_net.ipynb 26
+# %% ../nbs/03b_net.ipynb 28
 def urlsave(url, dest=None, reporthook=None, timeout=None):
     "Retrieve `url` and save based on its name"
     dest = urldest(url, dest)
     dest.parent.mkdir(parents=True, exist_ok=True)
     nm,msg = urlretrieve(url, dest, reporthook, timeout=timeout)
     return nm
 
-# %% ../nbs/03b_net.ipynb 28
+# %% ../nbs/03b_net.ipynb 30
 def urlvalid(x):
     "Test if `x` is a valid URL"
     return all (getattrs(urlparse(str(x)), 'scheme', 'netloc'))
 
-# %% ../nbs/03b_net.ipynb 30
+# %% ../nbs/03b_net.ipynb 32
 def urlrequest(url, verb, headers=None, route=None, query=None, data=None, json_data=True):
     "`Request` for `url` with optional route params replaced by `route`, plus `query` string, and post `data`"
     if route: url = url.format(**route)
     if query: url += '?' + urlencode(query)
     if isinstance(data,dict): data = (json.dumps if json_data else urlencode)(data).encode('ascii')
     return Request(url, headers=headers or {}, data=data or None, method=verb.upper())
 
-# %% ../nbs/03b_net.ipynb 33
+# %% ../nbs/03b_net.ipynb 35
 @patch
 def summary(self:Request, skip=None)->dict:
     "Summary containing full_url, headers, method, and data, removing `skip` from headers"
     res = L('full_url','method','data').map_dict(partial(getattr,self))
     res['headers'] = {k:v for k,v in self.headers.items() if k not in listify(skip)}
     return res
 
-# %% ../nbs/03b_net.ipynb 35
+# %% ../nbs/03b_net.ipynb 37
 def urlsend(url, verb, headers=None, route=None, query=None, data=None, json_data=True,
             return_json=True, return_headers=False, debug=None):
     "Send request with `urlrequest`, converting result to json if `return_json`"
@@ -213,7 +216,7 @@ def urlsend(url, verb, headers=None, route=None, query=None, data=None, json_dat
 
     return urlread(req, return_json=return_json, return_headers=return_headers)
 
-# %% ../nbs/03b_net.ipynb 36
+# %% ../nbs/03b_net.ipynb 38
 def do_request(url, post=False, headers=None, **data):
     "Call GET or json-encoded POST on `url`, depending on `post`"
     if data:
@@ -223,13 +226,13 @@ def do_request(url, post=False, headers=None, **data):
             data = None
     return urljson(Request(url, headers=headers, data=data or None))
 
-# %% ../nbs/03b_net.ipynb 37
+# %% ../nbs/03b_net.ipynb 39
 def _socket_det(port,host,dgram):
     if isinstance(port,int): family,addr = socket.AF_INET,(host or socket.gethostname(),port)
     else: family,addr = socket.AF_UNIX,port
     return family,addr,(socket.SOCK_STREAM,socket.SOCK_DGRAM)[dgram]
 
-# %% ../nbs/03b_net.ipynb 38
+# %% ../nbs/03b_net.ipynb 40
 def start_server(port, host=None, dgram=False, reuse_addr=True, n_queue=None):
     "Create a `socket` server on `port`, with optional `host`, of type `dgram`"
     listen_args = [n_queue] if n_queue else []
@@ -243,7 +246,7 @@ def start_server(port, host=None, dgram=False, reuse_addr=True, n_queue=None):
     s.listen(*listen_args)
     return s
 
-# %% ../nbs/03b_net.ipynb 40
+# %% ../nbs/03b_net.ipynb 42
 def start_client(port, host=None, dgram=False):
     "Create a `socket` client on `port`, with optional `host`, of type `dgram`"
     family,addr,typ = _socket_det(port,host,dgram)

nbs/03b_net.ipynb

@@ -18,12 +18,12 @@
     "#|export\n",
     "from fastcore.utils import *\n",
     "from fastcore.parallel import *\n",
-    "from functools import wraps\n",
     "\n",
+    "from functools import wraps\n",
     "import json,urllib,contextlib\n",
     "import socket,urllib.request,http,urllib\n",
     "from contextlib import contextmanager,ExitStack\n",
-    "from urllib.request import Request,urlretrieve,install_opener\n",
+    "from urllib.request import Request,urlretrieve,install_opener,HTTPErrorProcessor\n",
     "from urllib.error import HTTPError,URLError\n",
     "from urllib.parse import urlencode,urlparse,urlunparse\n",
     "from http.client import InvalidURL"
@@ -184,14 +184,24 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "### HTTP4xxClientError\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L63){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP4xxClientError\n",
        "\n",
        ">      HTTP4xxClientError (url, code, msg, hdrs, fp)\n",
        "\n",
        "Base class for client exceptions (code 4xx) from `url*` functions"
       ],
       "text/plain": [
-       "<nbdev.showdoc.BasicMarkdownRenderer>"
+       "---\n",
+       "\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L63){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP4xxClientError\n",
+       "\n",
+       ">      HTTP4xxClientError (url, code, msg, hdrs, fp)\n",
+       "\n",
+       "Base class for client exceptions (code 4xx) from `url*` functions"
       ]
      },
      "execution_count": null,
@@ -213,14 +223,24 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "### HTTP5xxServerError\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L68){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP5xxServerError\n",
        "\n",
        ">      HTTP5xxServerError (url, code, msg, hdrs, fp)\n",
        "\n",
        "Base class for server exceptions (code 5xx) from `url*` functions"
       ],
       "text/plain": [
-       "<nbdev.showdoc.BasicMarkdownRenderer>"
+       "---\n",
+       "\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L68){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP5xxServerError\n",
+       "\n",
+       ">      HTTP5xxServerError (url, code, msg, hdrs, fp)\n",
+       "\n",
+       "Base class for server exceptions (code 5xx) from `url*` functions"
       ]
      },
      "execution_count": null,
@@ -293,7 +313,43 @@
     "    if data is not None:\n",
     "        if not isinstance(data, (str,bytes)): data = urlencode(data)\n",
     "        if not isinstance(data, bytes): data = data.encode('ascii')\n",
-    "    return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)"
+    "    try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)\n",
+    "    except HTTPError as e: \n",
+    "        e.msg += f\"\\n====Error Body====\\n{e.read().decode(errors='ignore')}\"\n",
+    "        raise"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "With `urlopen`, the body of the response will also be returned in addition to the message if there is an error:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "404 Not Found\n",
+      "====Error Body====\n",
+      "{\n",
+      "  \"message\": \"Not Found\",\n",
+      "  \"documentation_url\": \"https://docs.github.com/rest\"\n",
+      "}\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "try: urlopen('https://api.github.com/v3')\n",
+    "except HTTPError as e: \n",
+    "    print(e.code, e.msg)\n",
+    "    assert 'documentation_url' in e.msg"
    ]
   },
   {
@@ -308,7 +364,7 @@
     "    try:\n",
     "        with urlopen(url, data=data, headers=headers, timeout=timeout, **kwargs) as u: res,hdrs = u.read(),u.headers\n",
     "    except HTTPError as e:\n",
-    "        if 400 <= e.code < 500: raise ExceptionsHTTP[e.code](e.url, e.hdrs, e.fp) from None\n",
+    "        if 400 <= e.code < 500: raise ExceptionsHTTP[e.code](e.url, e.hdrs, e.fp, msg=e.msg) from None\n",
     "        else: raise\n",
     "\n",
     "    if decode: res = res.decode()\n",