bug fixes

Author: theblackcat102

extractnet/metadata_extraction/json_ld.py

@@ -36,11 +36,13 @@ def extract_json(schema, metadata):
 
             if '@type' not in content:
                 continue
-            if isinstance(content["@type"], list):
+            if isinstance(content["@type"], list) and len(content["@type"]):
                 # some websites are using ['Person'] as type
                 content_type = content["@type"][0].lower()
-            else:
+            elif isinstance(content['@type'], str):
                 content_type = content["@type"].lower()
+            else:
+                continue
 
             if content_type in JSON_PUBLISHER_SCHEMA:
                 for candidate in ("name", "alternateName"):

extractnet/metadata_extraction/metadata.py

@@ -31,7 +31,7 @@
     TWITTER_ATTRS, METANAME_TAG, EXTRA_META, METANAME_TITLE
 )
 LOGGER = logging.getLogger(__name__)
-logging.getLogger('htmldate').setLevel(logging.WARNING)
+# logging.getLogger('htmldate').setLevel(logging.WARNING)
 
 
 def criteria_fulfilled(metadata):
@@ -298,8 +298,12 @@ def extract_author(tree):
             matches = tree.re_xpath("//*[re:match( text(), '{}' )]".format(text_author_pattern))
             if len(matches) > 0:
                 match_text = matches[0].text
-                author = re.search(text_author_pattern, match_text).group(0)
-                break
+                try:
+                    author = re.search(text_author_pattern, match_text).group(0)
+                except TypeError:
+                    continue
+                else:
+                    break
 
     return author
 
@@ -311,7 +315,9 @@ def extract_url(tree, default_url=None):
     url = default_url
     # try canonical link first
     element = tree.find('.//head//link[@rel="canonical"]')
-    if element is not None and URL_COMP_CHECK.match(element.attrib['href']):
+    if element is not None and \
+        'href' in element.attrib and \
+        URL_COMP_CHECK.match(element.attrib['href']):
         url = element.attrib['href']
     # try default language link
     else:
@@ -444,7 +450,10 @@ def extract_metadata(filecontent, default_url=None, date_config=None, fastmode=F
     date_config['url'] = metadata['url']
     metadata['date'] = find_date(tree, **date_config)
 
-    if metadata['sitename'] is not None:
+    if isinstance(metadata['sitename'], list):
+        metadata['sitename'] = metadata['sitename'][0]
+
+    if isinstance(metadata['sitename'], str):
         if metadata['sitename'].startswith('@'):
             # scrap Twitter ID
             metadata['sitename'] = re.sub(r'^@', '', metadata['sitename'])

extractnet/metadata_extraction/url_utils.py

@@ -1,3 +1,4 @@
+from multiprocessing.sharedctypes import Value
 import re
 from urllib.parse import ParseResult, parse_qs, urlencode, urlparse
 from tld import get_tld
@@ -57,7 +58,10 @@ def date_updater(url_date_token, date):
 
     month = url_date_token[1]
     if month > 0 and month < 13 and date.month != month:
-        date = date.replace(month = month)
+        try:
+            date = date.replace(month=month)
+        except ValueError: # when month=2
+            pass
 
     day = url_date_token[2]
     if day > 0 and day < 32 and day != date.day:

extractnet/metadata_extraction/video.py

@@ -77,7 +77,10 @@ def get_advance_fields(raw_html):
         if speechkit.find('iframe'):            
             if audio_urls == None:
                 audio_urls = []
-            audio_urls.append(speechkit_audio(speechkit.find('iframe').get('src')))
+            try:
+                audio_urls.append(speechkit_audio(speechkit.find('iframe').get('src')))
+            except ValueError:
+                pass
 
     '''
         Video extraction

extractnet/util.py

@@ -177,7 +177,10 @@ def fix_encoding(text):
     if isinstance(text, str):
         text = ftfy.fix_text(ftfy.fix_encoding(text))
         if '\\u' in text:
-            text = text.encode().decode('unicode_escape')
+            try:
+                text = text.encode().decode('unicode_escape')
+            except UnicodeDecodeError as e:
+                return text
         return text
     elif isinstance(text, list):
         return [ ftfy.fix_text(ftfy.fix_encoding(t)) for t in text ]