Merge pull request #73 from scrapfly/fix-bing

Fix TypeError in parse_serps when date field is None

Author: mazen-r

bing-scraper/bing.py

@@ -20,7 +20,6 @@
     "country": "GB",
     "proxy_pool": "public_residential_pool",
     "debug":True,
-    "os":"linux",
     "auto_scroll":True,
 }
 
@@ -37,7 +36,7 @@ def parse_serps(response: ScrapeApiResponse) -> List[Dict]:
         url = result.xpath(".//h2/a/@href").get()
         description = result.xpath("normalize-space(.//div/p)").extract_first()
         date = result.xpath(".//span[@class='news_dt']/text()").get()
-        if data is not None and len(date) > 12:
+        if data is not None and date is not None and len(date) > 12:
             date_pattern = re.compile(r"\b\d{2}-\d{2}-\d{4}\b")
             date_pattern.findall(description)
             dates = date_pattern.findall(date)

bing-scraper/results/keywords.json

@@ -1,18 +1,18 @@
 [
-  "email scraper",
-  "email scraping software",
-  "extract emails from web page",
-  "scrape emails from website free",
-  "scrape email addresses from websites",
-  "scrape emails from website",
-  "pull email addresses from website",
+  "free online email scraper",
+  "extract email from website free",
+  "extract emails from website",
+  "free email scraping tool",
+  "extract email address from website",
   "how to delete emails from website",
-  "email scraper",
-  "email scraping software",
-  "extract emails from web page",
-  "scrape emails from website free",
-  "scrape email addresses from websites",
-  "scrape emails from website",
+  "web email scraper",
   "pull email addresses from website",
-  "how to delete emails from website"
+  "free online email scraper",
+  "extract email from website free",
+  "extract emails from website",
+  "free email scraping tool",
+  "extract email address from website",
+  "how to delete emails from website",
+  "web email scraper",
+  "pull email addresses from website"
 ]