Handle none-responses from open_url

resources/lib/tokenresolver.py

@@ -199,7 +199,10 @@ def _get_xvrttoken(self, login_json=None):
         )
         data = dumps(payload).encode()
         headers = {'Content-Type': 'application/json', 'Cookie': login_cookie}
-        setcookie_header = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers).info().get('Set-Cookie')
+        response = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers)
+        if response is None:
+            return None
+        setcookie_header = response.info().get('Set-Cookie')
         xvrttoken = TokenResolver._create_token_dictionary(setcookie_header)
         if xvrttoken is None:
             return None
@@ -213,13 +216,22 @@ def _get_roaming_xvrttoken(self):
             return None
         cookie_value = 'vrtlogin-at=' + vrtlogin_at
         headers = {'Cookie': cookie_value}
-        req_info = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False).info()
+        response = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False)
+        if response is None:
+            return None
+        req_info = response.info()
         cookie_value += '; state=' + req_info.get('Set-Cookie').split('state=')[1].split('; ')[0]
-        url = open_url(req_info.get('Location'), follow_redirects=False).info().get('Location')
+        response = open_url(req_info.get('Location'), follow_redirects=False)
+        if response is None:
+            return None
+        url = response.info().get('Location')
         headers = {'Cookie': cookie_value}
         if url is None:
             return None
-        setcookie_header = open_url(url, headers=headers, follow_redirects=False).info().get('Set-Cookie')
+        response = open_url(url, headers=headers, follow_redirects=False)
+        if response is None:
+            return None
+        setcookie_header = response.info().get('Set-Cookie')
         return TokenResolver._create_token_dictionary(setcookie_header)
 
     def get_token(self, name, variant=None, url=None, roaming=False):

resources/lib/webscraper.py

@@ -31,6 +31,8 @@ def get_categories():
     if not valid_categories(categories):
         from bs4 import BeautifulSoup, SoupStrainer
         response = open_url('https://www.vrt.be/vrtnu/categorieen/')
+        if response is None:
+            return categories
         tiles = SoupStrainer('nui-list--content')
         soup = BeautifulSoup(response.read(), 'html.parser', parse_only=tiles)
 
@@ -79,10 +81,13 @@ def get_video_attributes(vrtnu_url):
     # Scrape video attributes
     from bs4 import BeautifulSoup, SoupStrainer
     try:
-        html_page = open_url(vrtnu_url, raise_errors='all').read()
+        response = open_url(vrtnu_url, raise_errors='all')
     except HTTPError as exc:
         log_error('Web scraping video attributes failed: {error}', error=exc)
         return None
+    if response is None:
+        return None
+    html_page = response.read()
     strainer = SoupStrainer(['section', 'div'], {'class': ['video-player', 'livestream__inner']})
     soup = BeautifulSoup(html_page, 'html.parser', parse_only=strainer)
     item = None