add body to error message

Author: hamelsmu

fastcore/net.py

@@ -18,11 +18,11 @@
 from .utils import *
 from .parallel import *
 from functools import wraps
-
+from pprint import pformat
 import json,urllib,contextlib
 import socket,urllib.request,http,urllib
 from contextlib import contextmanager,ExitStack
-from urllib.request import Request,urlretrieve,install_opener
+from urllib.request import Request,urlretrieve,install_opener,HTTPErrorProcessor
 from urllib.error import HTTPError,URLError
 from urllib.parse import urlencode,urlparse,urlunparse
 from http.client import InvalidURL
@@ -104,7 +104,11 @@ def urlopen(url, data=None, headers=None, timeout=None, **kwargs):
     if data is not None:
         if not isinstance(data, (str,bytes)): data = urlencode(data)
         if not isinstance(data, bytes): data = data.encode('ascii')
-    return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)
+    try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)
+    except HTTPError as e: 
+        e.msg += f"\n====Error Body====\n{pformat(json.loads(e.read()))}"
+        raise e
+    except Exception as e: raise e
 
 # %% ../nbs/03b_net.ipynb 18
 def urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs):
@@ -119,13 +123,13 @@ def urlread(url, data=None, headers=None, decode=True, return_json=False, return
     if return_json: res = loads(res)
     return (res,dict(hdrs)) if return_headers else res
 
-# %% ../nbs/03b_net.ipynb 19
+# %% ../nbs/03b_net.ipynb 20
 def urljson(url, data=None, timeout=None):
     "Retrieve `url` and decode json"
     res = urlread(url, data=data, timeout=timeout)
     return json.loads(res) if res else {}
 
-# %% ../nbs/03b_net.ipynb 21
+# %% ../nbs/03b_net.ipynb 22
 def urlcheck(url, timeout=10):
     if not url: return True
     try:
@@ -134,12 +138,12 @@ def urlcheck(url, timeout=10):
     except socket.timeout: return False
     except InvalidURL: return False
 
-# %% ../nbs/03b_net.ipynb 22
+# %% ../nbs/03b_net.ipynb 23
 def urlclean(url):
     "Remove fragment, params, and querystring from `url` if present"
     return urlunparse(urlparse(str(url))[:3]+('','',''))
 
-# %% ../nbs/03b_net.ipynb 24
+# %% ../nbs/03b_net.ipynb 25
 def urlretrieve(url, filename=None, reporthook=None, data=None, timeout=None):
     "Same as `urllib.request.urlretrieve` but also works with `Request` objects"
     with contextlib.closing(urlopen(url, data, timeout=timeout)) as fp:
@@ -165,43 +169,43 @@ def urlretrieve(url, filename=None, reporthook=None, data=None, timeout=None):
         raise ContentTooShortError(f"retrieval incomplete: got only {read} out of {size} bytes", headers)
     return filename,headers
 
-# %% ../nbs/03b_net.ipynb 25
+# %% ../nbs/03b_net.ipynb 26
 def urldest(url, dest=None):
     name = urlclean(Path(url).name)
     if dest is None: dest = name
     dest = Path(dest)
     return dest/name if dest.is_dir() else dest
 
-# %% ../nbs/03b_net.ipynb 26
+# %% ../nbs/03b_net.ipynb 27
 def urlsave(url, dest=None, reporthook=None, timeout=None):
     "Retrieve `url` and save based on its name"
     dest = urldest(url, dest)
     dest.parent.mkdir(parents=True, exist_ok=True)
     nm,msg = urlretrieve(url, dest, reporthook, timeout=timeout)
     return nm
 
-# %% ../nbs/03b_net.ipynb 28
+# %% ../nbs/03b_net.ipynb 29
 def urlvalid(x):
     "Test if `x` is a valid URL"
     return all (getattrs(urlparse(str(x)), 'scheme', 'netloc'))
 
-# %% ../nbs/03b_net.ipynb 30
+# %% ../nbs/03b_net.ipynb 31
 def urlrequest(url, verb, headers=None, route=None, query=None, data=None, json_data=True):
     "`Request` for `url` with optional route params replaced by `route`, plus `query` string, and post `data`"
     if route: url = url.format(**route)
     if query: url += '?' + urlencode(query)
     if isinstance(data,dict): data = (json.dumps if json_data else urlencode)(data).encode('ascii')
     return Request(url, headers=headers or {}, data=data or None, method=verb.upper())
 
-# %% ../nbs/03b_net.ipynb 33
+# %% ../nbs/03b_net.ipynb 34
 @patch
 def summary(self:Request, skip=None)->dict:
     "Summary containing full_url, headers, method, and data, removing `skip` from headers"
     res = L('full_url','method','data').map_dict(partial(getattr,self))
     res['headers'] = {k:v for k,v in self.headers.items() if k not in listify(skip)}
     return res
 
-# %% ../nbs/03b_net.ipynb 35
+# %% ../nbs/03b_net.ipynb 36
 def urlsend(url, verb, headers=None, route=None, query=None, data=None, json_data=True,
             return_json=True, return_headers=False, debug=None):
     "Send request with `urlrequest`, converting result to json if `return_json`"
@@ -213,7 +217,7 @@ def urlsend(url, verb, headers=None, route=None, query=None, data=None, json_dat
 
     return urlread(req, return_json=return_json, return_headers=return_headers)
 
-# %% ../nbs/03b_net.ipynb 36
+# %% ../nbs/03b_net.ipynb 37
 def do_request(url, post=False, headers=None, **data):
     "Call GET or json-encoded POST on `url`, depending on `post`"
     if data:
@@ -223,13 +227,13 @@ def do_request(url, post=False, headers=None, **data):
             data = None
     return urljson(Request(url, headers=headers, data=data or None))
 
-# %% ../nbs/03b_net.ipynb 37
+# %% ../nbs/03b_net.ipynb 38
 def _socket_det(port,host,dgram):
     if isinstance(port,int): family,addr = socket.AF_INET,(host or socket.gethostname(),port)
     else: family,addr = socket.AF_UNIX,port
     return family,addr,(socket.SOCK_STREAM,socket.SOCK_DGRAM)[dgram]
 
-# %% ../nbs/03b_net.ipynb 38
+# %% ../nbs/03b_net.ipynb 39
 def start_server(port, host=None, dgram=False, reuse_addr=True, n_queue=None):
     "Create a `socket` server on `port`, with optional `host`, of type `dgram`"
     listen_args = [n_queue] if n_queue else []
@@ -243,7 +247,7 @@ def start_server(port, host=None, dgram=False, reuse_addr=True, n_queue=None):
     s.listen(*listen_args)
     return s
 
-# %% ../nbs/03b_net.ipynb 40
+# %% ../nbs/03b_net.ipynb 41
 def start_client(port, host=None, dgram=False):
     "Create a `socket` client on `port`, with optional `host`, of type `dgram`"
     family,addr,typ = _socket_det(port,host,dgram)

nbs/03b_net.ipynb

@@ -19,11 +19,11 @@
     "from fastcore.utils import *\n",
     "from fastcore.parallel import *\n",
     "from functools import wraps\n",
-    "\n",
+    "from pprint import pformat\n",
     "import json,urllib,contextlib\n",
     "import socket,urllib.request,http,urllib\n",
     "from contextlib import contextmanager,ExitStack\n",
-    "from urllib.request import Request,urlretrieve,install_opener\n",
+    "from urllib.request import Request,urlretrieve,install_opener,HTTPErrorProcessor\n",
     "from urllib.error import HTTPError,URLError\n",
     "from urllib.parse import urlencode,urlparse,urlunparse\n",
     "from http.client import InvalidURL"
@@ -184,14 +184,24 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "### HTTP4xxClientError\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L64){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP4xxClientError\n",
        "\n",
        ">      HTTP4xxClientError (url, code, msg, hdrs, fp)\n",
        "\n",
        "Base class for client exceptions (code 4xx) from `url*` functions"
       ],
       "text/plain": [
-       "<nbdev.showdoc.BasicMarkdownRenderer>"
+       "---\n",
+       "\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L64){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP4xxClientError\n",
+       "\n",
+       ">      HTTP4xxClientError (url, code, msg, hdrs, fp)\n",
+       "\n",
+       "Base class for client exceptions (code 4xx) from `url*` functions"
       ]
      },
      "execution_count": null,
@@ -213,14 +223,24 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "### HTTP5xxServerError\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L69){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP5xxServerError\n",
        "\n",
        ">      HTTP5xxServerError (url, code, msg, hdrs, fp)\n",
        "\n",
        "Base class for server exceptions (code 5xx) from `url*` functions"
       ],
       "text/plain": [
-       "<nbdev.showdoc.BasicMarkdownRenderer>"
+       "---\n",
+       "\n",
+       "[source](https://github.com/fastai/fastcore/blob/master/fastcore/net.py#L69){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "\n",
+       "#### HTTP5xxServerError\n",
+       "\n",
+       ">      HTTP5xxServerError (url, code, msg, hdrs, fp)\n",
+       "\n",
+       "Base class for server exceptions (code 5xx) from `url*` functions"
       ]
      },
      "execution_count": null,
@@ -293,7 +313,11 @@
     "    if data is not None:\n",
     "        if not isinstance(data, (str,bytes)): data = urlencode(data)\n",
     "        if not isinstance(data, bytes): data = data.encode('ascii')\n",
-    "    return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)"
+    "    try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout)\n",
+    "    except HTTPError as e: \n",
+    "        e.msg += f\"\\n====Error Body====\\n{pformat(json.loads(e.read()))}\"\n",
+    "        raise e\n",
+    "    except Exception as e: raise e"
    ]
   },
   {
@@ -316,6 +340,17 @@
     "    return (res,dict(hdrs)) if return_headers else res"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The error should contain the body of the error, too\n",
+    "try: urlread('https://api.github.com/v3')\n",
+    "except HTTPError as e: assert 'documentation_url' in e.msg"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,