Skip to content

Commit 1e0da7b

Browse files
authored
Merge pull request #73 from scrapfly/fix-bing
Fix TypeError in parse_serps when date field is None
2 parents 7c6da77 + 7741b4e commit 1e0da7b

File tree

3 files changed

+96
-160
lines changed

3 files changed

+96
-160
lines changed

bing-scraper/bing.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
"country": "GB",
2121
"proxy_pool": "public_residential_pool",
2222
"debug":True,
23-
"os":"linux",
2423
"auto_scroll":True,
2524
}
2625

@@ -37,7 +36,7 @@ def parse_serps(response: ScrapeApiResponse) -> List[Dict]:
3736
url = result.xpath(".//h2/a/@href").get()
3837
description = result.xpath("normalize-space(.//div/p)").extract_first()
3938
date = result.xpath(".//span[@class='news_dt']/text()").get()
40-
if data is not None and len(date) > 12:
39+
if data is not None and date is not None and len(date) > 12:
4140
date_pattern = re.compile(r"\b\d{2}-\d{2}-\d{4}\b")
4241
date_pattern.findall(description)
4342
dates = date_pattern.findall(date)

bing-scraper/results/keywords.json

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
[
2-
"email scraper",
3-
"email scraping software",
4-
"extract emails from web page",
5-
"scrape emails from website free",
6-
"scrape email addresses from websites",
7-
"scrape emails from website",
8-
"pull email addresses from website",
2+
"free online email scraper",
3+
"extract email from website free",
4+
"extract emails from website",
5+
"free email scraping tool",
6+
"extract email address from website",
97
"how to delete emails from website",
10-
"email scraper",
11-
"email scraping software",
12-
"extract emails from web page",
13-
"scrape emails from website free",
14-
"scrape email addresses from websites",
15-
"scrape emails from website",
8+
"web email scraper",
169
"pull email addresses from website",
17-
"how to delete emails from website"
10+
"free online email scraper",
11+
"extract email from website free",
12+
"extract emails from website",
13+
"free email scraping tool",
14+
"extract email address from website",
15+
"how to delete emails from website",
16+
"web email scraper",
17+
"pull email addresses from website"
1818
]

0 commit comments

Comments
 (0)