fixes #168

Author: jph00

fastcore/_nbdev.py

@@ -141,6 +141,8 @@
          "join_path_file": "03_xtras.ipynb",
          "urlread": "03_xtras.ipynb",
          "urljson": "03_xtras.ipynb",
+         "urlwrap": "03_xtras.ipynb",
+         "urlcheck": "03_xtras.ipynb",
          "run": "03_xtras.ipynb",
          "do_request": "03_xtras.ipynb",
          "sort_by_run": "03_xtras.ipynb",

fastcore/xtras.py

@@ -2,9 +2,9 @@
 
 __all__ = ['dict2obj', 'tuplify', 'uniqueify', 'is_listy', 'shufflish', 'mapped', 'IterLen', 'ReindexCollection',
            'open_file', 'save_pickle', 'load_pickle', 'maybe_open', 'image_size', 'bunzip', 'join_path_file', 'urlread',
-           'urljson', 'run', 'do_request', 'sort_by_run', 'trace', 'round_multiple', 'modified_env', 'ContextManagers',
-           'str2bool', 'set_num_threads', 'ProcessPoolExecutor', 'ThreadPoolExecutor', 'parallel', 'run_procs',
-           'parallel_gen', 'threaded']
+           'urljson', 'urlwrap', 'urlcheck', 'run', 'do_request', 'sort_by_run', 'trace', 'round_multiple',
+           'modified_env', 'ContextManagers', 'str2bool', 'set_num_threads', 'ProcessPoolExecutor',
+           'ThreadPoolExecutor', 'parallel', 'run_procs', 'parallel_gen', 'threaded']
 
 # Cell
 from .imports import *
@@ -16,8 +16,9 @@
 from contextlib import contextmanager,ExitStack
 from pdb import set_trace
 from urllib.request import Request,urlopen
-from urllib.error import HTTPError
+from urllib.error import HTTPError,URLError
 from urllib.parse import urlencode
+from http.client import InvalidURL
 from threading import Thread
 
 # Cell
@@ -186,6 +187,25 @@ def urljson(url, data=None):
     "Retrieve `url` and decode json"
     return json.loads(urlread(url, data=data))
 
+# Cell
+_ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36'
+
+# Cell
+def urlwrap(url):
+    "Wrap `url` in a urllib `Request` with a user-agent header"
+    if not isinstance(url,Request): url = Request(url)
+    url.headers['User-Agent'] = _ua
+    return url
+
+# Cell
+def urlcheck(url, timeout=10):
+    if not url: return True
+    try:
+        with urlopen(urlwrap(url), timeout=timeout) as u: return u.status<400
+    except URLError: return False
+    except socket.timeout: return False
+    except InvalidURL: return False
+
 # Cell
 def run(cmd, *rest, ignore_ex=False, as_bytes=False):
     "Pass `cmd` (splitting with `shlex` if string) to `subprocess.run`; return `stdout`; raise `IOError` if fails"

nbs/03_xtras.ipynb

@@ -25,8 +25,9 @@
     "from contextlib import contextmanager,ExitStack\n",
     "from pdb import set_trace\n",
     "from urllib.request import Request,urlopen\n",
-    "from urllib.error import HTTPError\n",
+    "from urllib.error import HTTPError,URLError\n",
     "from urllib.parse import urlencode\n",
+    "from http.client import InvalidURL\n",
     "from threading import Thread"
    ]
   },
@@ -539,7 +540,7 @@
     {
      "data": {
       "text/plain": [
-       "['h', 'f', 'd', 'a', 'e', 'b', 'g', 'c']"
+       "['h', 'g', 'd', 'a', 'b', 'e', 'f', 'c']"
       ]
      },
      "execution_count": null,
@@ -1128,6 +1129,46 @@
     "    return json.loads(urlread(url, data=data))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "_ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "def urlwrap(url):\n",
+    "    \"Wrap `url` in a urllib `Request` with a user-agent header\"\n",
+    "    if not isinstance(url,Request): url = Request(url)\n",
+    "    url.headers['User-Agent'] = _ua\n",
+    "    return url"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "def urlcheck(url, timeout=10):\n",
+    "    if not url: return True\n",
+    "    try:\n",
+    "        with urlopen(urlwrap(url), timeout=timeout) as u: return u.status<400\n",
+    "    except URLError: return False\n",
+    "    except socket.timeout: return False\n",
+    "    except InvalidURL: return False"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,